21:04:07.780 [INFO - main] (KuduTestHarness.java:155) Creating a new MiniKuduCluster...
21:04:07.780 [INFO - main] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskRxIrUU/test-tmp
21:04:07.780 [INFO - main] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/asan/bin
21:04:07.780 [INFO - main] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskRxIrUU/java/build/dist-test/../../../build/asan/bin/kudu, test, mini_cluster, --serialization=pb]
21:04:07.782 [DEBUG - main] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 3
enable_kerberos: false
cluster_root: "/tmp/dist-test-taskRxIrUU/test-tmp/mini-kudu-cluster8143392791847762722"
extra_master_flags: "--txn_manager_enabled"
extra_master_flags: "--raft_heartbeat_interval_ms=100"
extra_tserver_flags: "--txn_keepalive_interval_ms=1000"
extra_tserver_flags: "--txn_staleness_tracker_interval_ms=250"
extra_tserver_flags: "--enable_txn_system_client_init=true"
hms_mode: NONE
mini_kdc_options {
}
principal: "kudu"
mini_oidc_options {
}
}
21:04:08.104 [DEBUG - main] (MiniKuduCluster.java:195) Response:
21:04:08.104 [DEBUG - main] (MiniKuduCluster.java:185) Request: start_cluster {
}
21:04:08.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:08.106463 23200 mini_oidc.cc:150] Starting JWKS server
21:04:08.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:08.108951 23200 webserver.cc:492] Webserver started at http://127.0.0.1:43617/ using document root <none> and password file <none>
21:04:08.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:08.109223 23200 mini_oidc.cc:186] Starting OIDC Discovery server
21:04:08.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:08.109769 23200 webserver.cc:492] Webserver started at http://127.0.0.1:37035/ using document root <none> and password file <none>
21:04:08.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-10-28T21:04:08Z chronyd version 4.6.1 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -NTS -SECHASH -IPV6 +DEBUG)
21:04:08.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-10-28T21:04:08Z Disabled control of system clock
21:04:08.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:08.140188 23200 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskRxIrUU/build/asan/bin/kudu
21:04:08.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskRxIrUU/build/asan/bin/kudu
21:04:08.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskRxIrUU/test-tmp/mini-kudu-cluster8143392791847762722/master-0/wal
21:04:08.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskRxIrUU/test-tmp/mini-kudu-cluster8143392791847762722/master-0/data
21:04:08.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
21:04:08.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
21:04:08.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
21:04:08.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:04:08.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:04:08.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:04:08.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskRxIrUU/test-tmp/mini-kudu-cluster8143392791847762722/master-0/logs
21:04:08.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskRxIrUU/test-tmp/mini-kudu-cluster8143392791847762722/master-0/data/info.pb
21:04:08.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:04:08.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
21:04:08.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
21:04:08.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
21:04:08.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
21:04:08.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:04:08.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:04:08.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:04:08.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
21:04:08.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
21:04:08.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
21:04:08.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
21:04:08.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.22.168.62:37239
21:04:08.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.22.168.62
21:04:08.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:04:08.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.22.168.20:37293
21:04:08.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:04:08.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:04:08.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:04:08.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
21:04:08.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.22.168.62:37239,127.22.168.61:32947,127.22.168.60:33247
21:04:08.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
21:04:08.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:43617/jwks.json
21:04:08.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled
21:04:08.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100 with env {}
21:04:08.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:08.453680 23211 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
21:04:08.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:08.454028 23211 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
21:04:08.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:08.454146 23211 flags.cc:425] Enabled unsafe flag: --never_fsync=true
21:04:08.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:08.462399 23211 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
21:04:08.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:08.462517 23211 flags.cc:425] Enabled experimental flag: --txn_manager_enabled=true
21:04:08.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:08.462596 23211 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
21:04:08.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:08.462651 23211 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
21:04:08.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:08.462747 23211 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
21:04:08.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:08.462810 23211 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
21:04:08.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:08.462880 23211 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:43617/jwks.json
21:04:08.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:08.472692 23211 master_runner.cc:386] Master server non-default flags:
21:04:08.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:04:08.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.22.168.20:37293
21:04:08.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:04:08.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:04:08.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100
21:04:08.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskRxIrUU/test-tmp/mini-kudu-cluster8143392791847762722/master-0/data
21:04:08.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskRxIrUU/test-tmp/mini-kudu-cluster8143392791847762722/master-0/wal
21:04:08.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
21:04:08.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.22.168.62:37239,127.22.168.61:32947,127.22.168.60:33247
21:04:08.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled=true
21:04:08.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:04:08.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:04:08.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
21:04:08.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.22.168.62:37239
21:04:08.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
21:04:08.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
21:04:08.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
21:04:08.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:43617/jwks.json
21:04:08.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:04:08.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:04:08.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskRxIrUU/test-tmp/mini-kudu-cluster8143392791847762722/master-0/data/info.pb
21:04:08.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.22.168.62
21:04:08.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:04:08.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
21:04:08.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:04:08.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
21:04:08.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
21:04:08.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:04:08.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskRxIrUU/test-tmp/mini-kudu-cluster8143392791847762722/master-0/logs
21:04:08.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:04:08.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
21:04:08.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
21:04:08.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
21:04:08.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
21:04:08.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 60f5e5267b92c39485a66121d3ce3cc7ef57b0e0
21:04:08.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
21:04:08.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 28 Oct 2025 20:43:24 UTC on 24a791456cd2
21:04:08.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 8764
21:04:08.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
21:04:08.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:08.474656 23211 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
21:04:08.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:08.476565 23211 file_cache.cc:492] Constructed file cache file cache with capacity 419430
21:04:08.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:08.484551 23216 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:04:08.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:08.484880 23217 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:04:08.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:08.485420 23219 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:04:08.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:08.486370 23211 server_base.cc:1047] running on GCE node
21:04:08.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:08.487129 23211 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
21:04:08.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:08.488574 23211 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
21:04:08.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:08.489804 23211 hybrid_clock.cc:648] HybridClock initialized: now 1761685448489735 us; error 42 us; skew 500 ppm
21:04:08.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:08.490171 23211 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
21:04:08.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:08.492517 23211 webserver.cc:492] Webserver started at http://127.22.168.62:33359/ using document root <none> and password file <none>
21:04:08.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:08.493252 23211 fs_manager.cc:362] Metadata directory not provided
21:04:08.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:08.493377 23211 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
21:04:08.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:08.493731 23211 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
21:04:08.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:08.496095 23211 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskRxIrUU/test-tmp/mini-kudu-cluster8143392791847762722/master-0/data/instance:
21:04:08.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "e9dd218ca2244f5a934aac2b7e58eb1d"
21:04:08.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-28 21:04:08 on dist-test-slave-vx1b"
21:04:08.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:08.496893 23211 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskRxIrUU/test-tmp/mini-kudu-cluster8143392791847762722/master-0/wal/instance:
21:04:08.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "e9dd218ca2244f5a934aac2b7e58eb1d"
21:04:08.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-28 21:04:08 on dist-test-slave-vx1b"
21:04:08.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:08.502178 23211 fs_manager.cc:696] Time spent creating directory manager: real 0.005s user 0.001s sys 0.003s
21:04:08.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:08.505455 23225 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
21:04:08.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:08.506690 23211 fs_manager.cc:730] Time spent opening block manager: real 0.003s user 0.002s sys 0.001s
21:04:08.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:08.506852 23211 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskRxIrUU/test-tmp/mini-kudu-cluster8143392791847762722/master-0/data,/tmp/dist-test-taskRxIrUU/test-tmp/mini-kudu-cluster8143392791847762722/master-0/wal
21:04:08.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "e9dd218ca2244f5a934aac2b7e58eb1d"
21:04:08.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-28 21:04:08 on dist-test-slave-vx1b"
21:04:08.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:08.507054 23211 fs_report.cc:389] FS layout report
21:04:08.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
21:04:08.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskRxIrUU/test-tmp/mini-kudu-cluster8143392791847762722/master-0/wal
21:04:08.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskRxIrUU/test-tmp/mini-kudu-cluster8143392791847762722/master-0/wal
21:04:08.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskRxIrUU/test-tmp/mini-kudu-cluster8143392791847762722/master-0/data/data
21:04:08.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
21:04:08.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
21:04:08.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
21:04:08.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
21:04:08.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
21:04:08.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
21:04:08.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
21:04:08.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
21:04:08.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
21:04:08.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
21:04:08.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
21:04:08.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:08.550848 23211 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
21:04:08.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:08.551781 23211 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
21:04:08.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:08.552106 23211 kserver.cc:163] Server-wide thread pool size limit: 3276
21:04:08.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:08.574878 23211 rpc_server.cc:307] RPC server started. Bound to: 127.22.168.62:37239
21:04:08.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:08.574921 23286 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.22.168.62:37239 every 8 connection(s)
21:04:08.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:08.576471 23211 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskRxIrUU/test-tmp/mini-kudu-cluster8143392791847762722/master-0/data/info.pb
21:04:08.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:08.580724 23287 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
21:04:08.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:08.586491 23200 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskRxIrUU/build/asan/bin/kudu as pid 23211
21:04:08.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:08.586817 23200 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskRxIrUU/test-tmp/mini-kudu-cluster8143392791847762722/master-0/wal/instance
21:04:08.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:08.586633 23287 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.22.168.62" port: 37239 } has no permanent_uuid. Determining permanent_uuid...
21:04:08.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:08.589187 23200 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskRxIrUU/build/asan/bin/kudu
21:04:08.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskRxIrUU/build/asan/bin/kudu
21:04:08.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskRxIrUU/test-tmp/mini-kudu-cluster8143392791847762722/master-1/wal
21:04:08.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskRxIrUU/test-tmp/mini-kudu-cluster8143392791847762722/master-1/data
21:04:08.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
21:04:08.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
21:04:08.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
21:04:08.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:04:08.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:04:08.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:04:08.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskRxIrUU/test-tmp/mini-kudu-cluster8143392791847762722/master-1/logs
21:04:08.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskRxIrUU/test-tmp/mini-kudu-cluster8143392791847762722/master-1/data/info.pb
21:04:08.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:04:08.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
21:04:08.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
21:04:08.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
21:04:08.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
21:04:08.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:04:08.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:04:08.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:04:08.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
21:04:08.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
21:04:08.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
21:04:08.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
21:04:08.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.22.168.61:32947
21:04:08.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.22.168.61
21:04:08.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:04:08.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.22.168.20:37293
21:04:08.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:04:08.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:04:08.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:04:08.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
21:04:08.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.22.168.62:37239,127.22.168.61:32947,127.22.168.60:33247
21:04:08.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
21:04:08.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:43617/jwks.json
21:04:08.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled
21:04:08.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100 with env {}
21:04:08.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:08.607693 23287 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.22.168.61" port: 32947 } has no permanent_uuid. Determining permanent_uuid...
21:04:08.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:08.608922 23229 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.22.168.61:32947: connect: Connection refused (error 111)
21:04:08.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:08.610476 23287 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.22.168.61:32947: Network error: Client connection negotiation failed: client connection to 127.22.168.61:32947: connect: Connection refused (error 111)
21:04:08.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:08.660842 23287 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.22.168.61" port: 32947 } attempt: 1
21:04:08.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:08.662781 23287 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.22.168.61:32947: Network error: Client connection negotiation failed: client connection to 127.22.168.61:32947: connect: Connection refused (error 111)
21:04:08.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:08.731033 23287 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.22.168.61" port: 32947 } attempt: 2
21:04:08.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:08.732933 23287 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.22.168.61:32947: Network error: Client connection negotiation failed: client connection to 127.22.168.61:32947: connect: Connection refused (error 111)
21:04:08.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:08.824226 23287 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.22.168.61" port: 32947 } attempt: 3
21:04:08.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:08.826233 23287 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.22.168.61:32947: Network error: Client connection negotiation failed: client connection to 127.22.168.61:32947: connect: Connection refused (error 111)
21:04:08.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:08.913084 23291 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
21:04:08.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:08.913395 23291 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
21:04:08.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:08.913471 23291 flags.cc:425] Enabled unsafe flag: --never_fsync=true
21:04:08.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:08.921470 23291 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
21:04:08.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:08.921563 23291 flags.cc:425] Enabled experimental flag: --txn_manager_enabled=true
21:04:08.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:08.921615 23291 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
21:04:08.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:08.921655 23291 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
21:04:08.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:08.921706 23291 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
21:04:08.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:08.921746 23291 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
21:04:08.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:08.921788 23291 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:43617/jwks.json
21:04:08.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:08.931540 23291 master_runner.cc:386] Master server non-default flags:
21:04:08.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:04:08.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.22.168.20:37293
21:04:08.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:04:08.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:04:08.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100
21:04:08.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskRxIrUU/test-tmp/mini-kudu-cluster8143392791847762722/master-1/data
21:04:08.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskRxIrUU/test-tmp/mini-kudu-cluster8143392791847762722/master-1/wal
21:04:08.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
21:04:08.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.22.168.62:37239,127.22.168.61:32947,127.22.168.60:33247
21:04:08.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled=true
21:04:08.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:04:08.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:04:08.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
21:04:08.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.22.168.61:32947
21:04:08.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
21:04:08.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
21:04:08.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
21:04:08.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:43617/jwks.json
21:04:08.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:04:08.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:04:08.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskRxIrUU/test-tmp/mini-kudu-cluster8143392791847762722/master-1/data/info.pb
21:04:08.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.22.168.61
21:04:08.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:04:08.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
21:04:08.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:04:08.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
21:04:08.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
21:04:08.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:04:08.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskRxIrUU/test-tmp/mini-kudu-cluster8143392791847762722/master-1/logs
21:04:08.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:04:08.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
21:04:08.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
21:04:08.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
21:04:08.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
21:04:08.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 60f5e5267b92c39485a66121d3ce3cc7ef57b0e0
21:04:08.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
21:04:08.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 28 Oct 2025 20:43:24 UTC on 24a791456cd2
21:04:08.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 8764
21:04:08.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
21:04:08.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:08.933173 23291 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
21:04:08.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:08.934907 23291 file_cache.cc:492] Constructed file cache file cache with capacity 419430
21:04:08.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:08.942718 23297 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:04:08.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:08.943135 23300 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:04:08.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:08.943357 23298 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:04:08.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:08.944274 23291 server_base.cc:1047] running on GCE node
21:04:08.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:08.945039 23291 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
21:04:08.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:08.946336 23291 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
21:04:08.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:08.947540 23291 hybrid_clock.cc:648] HybridClock initialized: now 1761685448947488 us; error 32 us; skew 500 ppm
21:04:08.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:08.947821 23291 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
21:04:08.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:08.949822 23291 webserver.cc:492] Webserver started at http://127.22.168.61:37179/ using document root <none> and password file <none>
21:04:08.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:08.950452 23291 fs_manager.cc:362] Metadata directory not provided
21:04:08.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:08.950546 23291 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
21:04:08.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:08.950819 23291 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
21:04:08.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:08.953073 23291 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskRxIrUU/test-tmp/mini-kudu-cluster8143392791847762722/master-1/data/instance:
21:04:08.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "c520077acf3341e29447c6c2b8d3f74c"
21:04:08.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-28 21:04:08 on dist-test-slave-vx1b"
21:04:08.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:08.953729 23291 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskRxIrUU/test-tmp/mini-kudu-cluster8143392791847762722/master-1/wal/instance:
21:04:08.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "c520077acf3341e29447c6c2b8d3f74c"
21:04:08.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-28 21:04:08 on dist-test-slave-vx1b"
21:04:08.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:08.958578 23291 fs_manager.cc:696] Time spent creating directory manager: real 0.005s user 0.005s sys 0.000s
21:04:08.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:08.961661 23306 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
21:04:08.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:08.962844 23291 fs_manager.cc:730] Time spent opening block manager: real 0.003s user 0.003s sys 0.000s
21:04:08.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:08.962976 23291 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskRxIrUU/test-tmp/mini-kudu-cluster8143392791847762722/master-1/data,/tmp/dist-test-taskRxIrUU/test-tmp/mini-kudu-cluster8143392791847762722/master-1/wal
21:04:08.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "c520077acf3341e29447c6c2b8d3f74c"
21:04:08.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-28 21:04:08 on dist-test-slave-vx1b"
21:04:08.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:08.963120 23291 fs_report.cc:389] FS layout report
21:04:08.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
21:04:08.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskRxIrUU/test-tmp/mini-kudu-cluster8143392791847762722/master-1/wal
21:04:08.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskRxIrUU/test-tmp/mini-kudu-cluster8143392791847762722/master-1/wal
21:04:08.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskRxIrUU/test-tmp/mini-kudu-cluster8143392791847762722/master-1/data/data
21:04:08.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
21:04:08.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
21:04:08.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
21:04:08.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
21:04:08.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
21:04:08.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
21:04:08.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
21:04:08.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
21:04:08.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
21:04:08.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
21:04:08.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
21:04:08.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:08.969447 23287 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.22.168.61" port: 32947 } attempt: 4
21:04:08.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:08.971148 23287 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.22.168.61:32947: Network error: Client connection negotiation failed: client connection to 127.22.168.61:32947: connect: Connection refused (error 111)
21:04:08.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:08.983157 23291 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
21:04:08.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:08.983947 23291 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
21:04:08.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:08.984181 23291 kserver.cc:163] Server-wide thread pool size limit: 3276
21:04:09.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:09.006449 23291 rpc_server.cc:307] RPC server started. Bound to: 127.22.168.61:32947
21:04:09.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:09.006539 23367 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.22.168.61:32947 every 8 connection(s)
21:04:09.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:09.007905 23291 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskRxIrUU/test-tmp/mini-kudu-cluster8143392791847762722/master-1/data/info.pb
21:04:09.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:09.011976 23368 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
21:04:09.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:09.016819 23200 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskRxIrUU/build/asan/bin/kudu as pid 23291
21:04:09.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:09.017024 23200 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskRxIrUU/test-tmp/mini-kudu-cluster8143392791847762722/master-1/wal/instance
21:04:09.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:09.018267 23200 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskRxIrUU/build/asan/bin/kudu
21:04:09.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskRxIrUU/build/asan/bin/kudu
21:04:09.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskRxIrUU/test-tmp/mini-kudu-cluster8143392791847762722/master-2/wal
21:04:09.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskRxIrUU/test-tmp/mini-kudu-cluster8143392791847762722/master-2/data
21:04:09.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
21:04:09.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
21:04:09.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
21:04:09.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:04:09.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:04:09.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:04:09.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskRxIrUU/test-tmp/mini-kudu-cluster8143392791847762722/master-2/logs
21:04:09.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskRxIrUU/test-tmp/mini-kudu-cluster8143392791847762722/master-2/data/info.pb
21:04:09.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:04:09.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
21:04:09.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
21:04:09.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
21:04:09.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
21:04:09.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:04:09.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:04:09.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:04:09.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
21:04:09.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
21:04:09.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
21:04:09.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
21:04:09.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.22.168.60:33247
21:04:09.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.22.168.60
21:04:09.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:04:09.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.22.168.20:37293
21:04:09.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:04:09.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:04:09.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:04:09.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
21:04:09.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.22.168.62:37239,127.22.168.61:32947,127.22.168.60:33247
21:04:09.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
21:04:09.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:43617/jwks.json
21:04:09.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled
21:04:09.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100 with env {}
21:04:09.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:09.018095 23368 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.22.168.62" port: 37239 } has no permanent_uuid. Determining permanent_uuid...
21:04:09.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:09.030628 23368 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.22.168.61" port: 32947 } has no permanent_uuid. Determining permanent_uuid...
21:04:09.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:09.039422 23368 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.22.168.60" port: 33247 } has no permanent_uuid. Determining permanent_uuid...
21:04:09.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:09.040413 23310 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.22.168.60:33247: connect: Connection refused (error 111)
21:04:09.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:09.041903 23368 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.22.168.60:33247: Network error: Client connection negotiation failed: client connection to 127.22.168.60:33247: connect: Connection refused (error 111)
21:04:09.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:09.091202 23368 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.22.168.60" port: 33247 } attempt: 1
21:04:09.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:09.093225 23368 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.22.168.60:33247: Network error: Client connection negotiation failed: client connection to 127.22.168.60:33247: connect: Connection refused (error 111)
21:04:09.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:09.161433 23368 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.22.168.60" port: 33247 } attempt: 2
21:04:09.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:09.163479 23368 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.22.168.60:33247: Network error: Client connection negotiation failed: client connection to 127.22.168.60:33247: connect: Connection refused (error 111)
21:04:09.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:09.254711 23368 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.22.168.60" port: 33247 } attempt: 3
21:04:09.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:09.256747 23368 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.22.168.60:33247: Network error: Client connection negotiation failed: client connection to 127.22.168.60:33247: connect: Connection refused (error 111)
21:04:09.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:09.270413 23287 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.22.168.61" port: 32947 } attempt: 5
21:04:09.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:09.277309 23287 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.22.168.60" port: 33247 } has no permanent_uuid. Determining permanent_uuid...
21:04:09.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:09.279145 23287 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.22.168.60:33247: Network error: Client connection negotiation failed: client connection to 127.22.168.60:33247: connect: Connection refused (error 111)
21:04:09.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:09.330381 23287 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.22.168.60" port: 33247 } attempt: 1
21:04:09.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:09.332325 23287 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.22.168.60:33247: Network error: Client connection negotiation failed: client connection to 127.22.168.60:33247: connect: Connection refused (error 111)
21:04:09.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:09.339517 23371 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
21:04:09.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:09.339881 23371 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
21:04:09.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:09.340032 23371 flags.cc:425] Enabled unsafe flag: --never_fsync=true
21:04:09.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:09.348238 23371 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
21:04:09.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:09.348352 23371 flags.cc:425] Enabled experimental flag: --txn_manager_enabled=true
21:04:09.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:09.348425 23371 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
21:04:09.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:09.348506 23371 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
21:04:09.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:09.348588 23371 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
21:04:09.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:09.348651 23371 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
21:04:09.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:09.348721 23371 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:43617/jwks.json
21:04:09.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:09.358719 23371 master_runner.cc:386] Master server non-default flags:
21:04:09.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:04:09.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.22.168.20:37293
21:04:09.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:04:09.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:04:09.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100
21:04:09.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskRxIrUU/test-tmp/mini-kudu-cluster8143392791847762722/master-2/data
21:04:09.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskRxIrUU/test-tmp/mini-kudu-cluster8143392791847762722/master-2/wal
21:04:09.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
21:04:09.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.22.168.62:37239,127.22.168.61:32947,127.22.168.60:33247
21:04:09.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled=true
21:04:09.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:04:09.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:04:09.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
21:04:09.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.22.168.60:33247
21:04:09.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
21:04:09.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
21:04:09.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
21:04:09.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:43617/jwks.json
21:04:09.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:04:09.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:04:09.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskRxIrUU/test-tmp/mini-kudu-cluster8143392791847762722/master-2/data/info.pb
21:04:09.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.22.168.60
21:04:09.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:04:09.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
21:04:09.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:04:09.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
21:04:09.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
21:04:09.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:04:09.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskRxIrUU/test-tmp/mini-kudu-cluster8143392791847762722/master-2/logs
21:04:09.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:04:09.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
21:04:09.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
21:04:09.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
21:04:09.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
21:04:09.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 60f5e5267b92c39485a66121d3ce3cc7ef57b0e0
21:04:09.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
21:04:09.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 28 Oct 2025 20:43:24 UTC on 24a791456cd2
21:04:09.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 8764
21:04:09.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
21:04:09.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:09.360831 23371 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
21:04:09.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:09.362738 23371 file_cache.cc:492] Constructed file cache file cache with capacity 419430
21:04:09.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:09.370826 23378 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:04:09.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:09.370793 23379 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:04:09.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:09.371570 23381 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:04:09.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:09.372386 23371 server_base.cc:1047] running on GCE node
21:04:09.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:09.373155 23371 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
21:04:09.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:09.374601 23371 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
21:04:09.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:09.375849 23371 hybrid_clock.cc:648] HybridClock initialized: now 1761685449375770 us; error 56 us; skew 500 ppm
21:04:09.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:09.376195 23371 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
21:04:09.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:09.378715 23371 webserver.cc:492] Webserver started at http://127.22.168.60:33853/ using document root <none> and password file <none>
21:04:09.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:09.379449 23371 fs_manager.cc:362] Metadata directory not provided
21:04:09.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:09.379575 23371 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
21:04:09.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:09.379894 23371 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
21:04:09.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:09.382443 23371 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskRxIrUU/test-tmp/mini-kudu-cluster8143392791847762722/master-2/data/instance:
21:04:09.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "eb664add87f44013b596f4cc01c9a41c"
21:04:09.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-28 21:04:09 on dist-test-slave-vx1b"
21:04:09.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:09.383324 23371 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskRxIrUU/test-tmp/mini-kudu-cluster8143392791847762722/master-2/wal/instance:
21:04:09.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "eb664add87f44013b596f4cc01c9a41c"
21:04:09.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-28 21:04:09 on dist-test-slave-vx1b"
21:04:09.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:09.389089 23371 fs_manager.cc:696] Time spent creating directory manager: real 0.005s user 0.005s sys 0.003s
21:04:09.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:09.392642 23387 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
21:04:09.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:09.394292 23371 fs_manager.cc:730] Time spent opening block manager: real 0.003s user 0.004s sys 0.000s
21:04:09.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:09.394447 23371 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskRxIrUU/test-tmp/mini-kudu-cluster8143392791847762722/master-2/data,/tmp/dist-test-taskRxIrUU/test-tmp/mini-kudu-cluster8143392791847762722/master-2/wal
21:04:09.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "eb664add87f44013b596f4cc01c9a41c"
21:04:09.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-28 21:04:09 on dist-test-slave-vx1b"
21:04:09.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:09.394620 23371 fs_report.cc:389] FS layout report
21:04:09.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
21:04:09.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskRxIrUU/test-tmp/mini-kudu-cluster8143392791847762722/master-2/wal
21:04:09.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskRxIrUU/test-tmp/mini-kudu-cluster8143392791847762722/master-2/wal
21:04:09.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskRxIrUU/test-tmp/mini-kudu-cluster8143392791847762722/master-2/data/data
21:04:09.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
21:04:09.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
21:04:09.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
21:04:09.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
21:04:09.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
21:04:09.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
21:04:09.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
21:04:09.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
21:04:09.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
21:04:09.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
21:04:09.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
21:04:09.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:09.399979 23368 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.22.168.60" port: 33247 } attempt: 4
21:04:09.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:09.400509 23287 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.22.168.60" port: 33247 } attempt: 2
21:04:09.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:09.402148 23368 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.22.168.60:33247: Network error: Client connection negotiation failed: client connection to 127.22.168.60:33247: connect: Connection refused (error 111)
21:04:09.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:09.402760 23287 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.22.168.60:33247: Network error: Client connection negotiation failed: client connection to 127.22.168.60:33247: connect: Connection refused (error 111)
21:04:09.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:09.419745 23371 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
21:04:09.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:09.420665 23371 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
21:04:09.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:09.420929 23371 kserver.cc:163] Server-wide thread pool size limit: 3276
21:04:09.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:09.444016 23371 rpc_server.cc:307] RPC server started. Bound to: 127.22.168.60:33247
21:04:09.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:09.444097 23448 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.22.168.60:33247 every 8 connection(s)
21:04:09.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:09.445541 23371 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskRxIrUU/test-tmp/mini-kudu-cluster8143392791847762722/master-2/data/info.pb
21:04:09.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:09.450001 23449 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
21:04:09.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:09.455025 23200 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskRxIrUU/build/asan/bin/kudu as pid 23371
21:04:09.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:09.455227 23200 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskRxIrUU/test-tmp/mini-kudu-cluster8143392791847762722/master-2/wal/instance
21:04:09.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:09.456483 23449 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.22.168.62" port: 37239 } has no permanent_uuid. Determining permanent_uuid...
21:04:09.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:09.471586 23449 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.22.168.61" port: 32947 } has no permanent_uuid. Determining permanent_uuid...
21:04:09.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:09.478919 23449 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.22.168.60" port: 33247 } has no permanent_uuid. Determining permanent_uuid...
21:04:09.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:09.491443 23449 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P eb664add87f44013b596f4cc01c9a41c: Bootstrap starting.
21:04:09.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:09.495165 23449 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P eb664add87f44013b596f4cc01c9a41c: Neither blocks nor log segments found. Creating new log.
21:04:09.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:09.496310 23449 log.cc:826] T 00000000000000000000000000000000 P eb664add87f44013b596f4cc01c9a41c: Log is configured to *not* fsync() on all Append() calls
21:04:09.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:09.499192 23449 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P eb664add87f44013b596f4cc01c9a41c: No bootstrap required, opened a new log
21:04:09.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:09.504473 23449 raft_consensus.cc:359] T 00000000000000000000000000000000 P eb664add87f44013b596f4cc01c9a41c [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e9dd218ca2244f5a934aac2b7e58eb1d" member_type: VOTER last_known_addr { host: "127.22.168.62" port: 37239 } } peers { permanent_uuid: "c520077acf3341e29447c6c2b8d3f74c" member_type: VOTER last_known_addr { host: "127.22.168.61" port: 32947 } } peers { permanent_uuid: "eb664add87f44013b596f4cc01c9a41c" member_type: VOTER last_known_addr { host: "127.22.168.60" port: 33247 } }
21:04:09.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:09.504877 23449 raft_consensus.cc:385] T 00000000000000000000000000000000 P eb664add87f44013b596f4cc01c9a41c [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
21:04:09.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:09.505015 23449 raft_consensus.cc:740] T 00000000000000000000000000000000 P eb664add87f44013b596f4cc01c9a41c [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: eb664add87f44013b596f4cc01c9a41c, State: Initialized, Role: FOLLOWER
21:04:09.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:09.505661 23449 consensus_queue.cc:260] T 00000000000000000000000000000000 P eb664add87f44013b596f4cc01c9a41c [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e9dd218ca2244f5a934aac2b7e58eb1d" member_type: VOTER last_known_addr { host: "127.22.168.62" port: 37239 } } peers { permanent_uuid: "c520077acf3341e29447c6c2b8d3f74c" member_type: VOTER last_known_addr { host: "127.22.168.61" port: 32947 } } peers { permanent_uuid: "eb664add87f44013b596f4cc01c9a41c" member_type: VOTER last_known_addr { host: "127.22.168.60" port: 33247 } }
21:04:09.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:09.506592 23458 sys_catalog.cc:455] T 00000000000000000000000000000000 P eb664add87f44013b596f4cc01c9a41c [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e9dd218ca2244f5a934aac2b7e58eb1d" member_type: VOTER last_known_addr { host: "127.22.168.62" port: 37239 } } peers { permanent_uuid: "c520077acf3341e29447c6c2b8d3f74c" member_type: VOTER last_known_addr { host: "127.22.168.61" port: 32947 } } peers { permanent_uuid: "eb664add87f44013b596f4cc01c9a41c" member_type: VOTER last_known_addr { host: "127.22.168.60" port: 33247 } } }
21:04:09.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:09.506826 23458 sys_catalog.cc:458] T 00000000000000000000000000000000 P eb664add87f44013b596f4cc01c9a41c [sys.catalog]: This master's current role is: FOLLOWER
21:04:09.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:09.507675 23449 sys_catalog.cc:565] T 00000000000000000000000000000000 P eb664add87f44013b596f4cc01c9a41c [sys.catalog]: configured and running, proceeding with master startup.
21:04:09.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:09.508978 23287 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.22.168.60" port: 33247 } attempt: 3
21:04:09.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:09.519204 23469 catalog_manager.cc:1568] T 00000000000000000000000000000000 P eb664add87f44013b596f4cc01c9a41c: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
21:04:09.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:09.519330 23469 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
21:04:09.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:09.523491 23287 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P e9dd218ca2244f5a934aac2b7e58eb1d: Bootstrap starting.
21:04:09.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:09.516615 23371 master_runner.cc:186] Error getting master registration for 127.22.168.62:37239: OK, instance_id { permanent_uuid: "e9dd218ca2244f5a934aac2b7e58eb1d" instance_seqno: 1761685448569562 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
21:04:09.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:09.523761 23371 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
21:04:09.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:09.527262 23287 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P e9dd218ca2244f5a934aac2b7e58eb1d: Neither blocks nor log segments found. Creating new log.
21:04:09.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:09.528379 23287 log.cc:826] T 00000000000000000000000000000000 P e9dd218ca2244f5a934aac2b7e58eb1d: Log is configured to *not* fsync() on all Append() calls
21:04:09.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:09.530982 23287 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P e9dd218ca2244f5a934aac2b7e58eb1d: No bootstrap required, opened a new log
21:04:09.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:09.535485 23458 raft_consensus.cc:493] T 00000000000000000000000000000000 P eb664add87f44013b596f4cc01c9a41c [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
21:04:09.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:09.535776 23458 raft_consensus.cc:515] T 00000000000000000000000000000000 P eb664add87f44013b596f4cc01c9a41c [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e9dd218ca2244f5a934aac2b7e58eb1d" member_type: VOTER last_known_addr { host: "127.22.168.62" port: 37239 } } peers { permanent_uuid: "c520077acf3341e29447c6c2b8d3f74c" member_type: VOTER last_known_addr { host: "127.22.168.61" port: 32947 } } peers { permanent_uuid: "eb664add87f44013b596f4cc01c9a41c" member_type: VOTER last_known_addr { host: "127.22.168.60" port: 33247 } }
21:04:09.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:09.537688 23458 leader_election.cc:290] T 00000000000000000000000000000000 P eb664add87f44013b596f4cc01c9a41c [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers e9dd218ca2244f5a934aac2b7e58eb1d (127.22.168.62:37239), c520077acf3341e29447c6c2b8d3f74c (127.22.168.61:32947)
21:04:09.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:09.537808 23287 raft_consensus.cc:359] T 00000000000000000000000000000000 P e9dd218ca2244f5a934aac2b7e58eb1d [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e9dd218ca2244f5a934aac2b7e58eb1d" member_type: VOTER last_known_addr { host: "127.22.168.62" port: 37239 } } peers { permanent_uuid: "c520077acf3341e29447c6c2b8d3f74c" member_type: VOTER last_known_addr { host: "127.22.168.61" port: 32947 } } peers { permanent_uuid: "eb664add87f44013b596f4cc01c9a41c" member_type: VOTER last_known_addr { host: "127.22.168.60" port: 33247 } }
21:04:09.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:09.538206 23287 raft_consensus.cc:385] T 00000000000000000000000000000000 P e9dd218ca2244f5a934aac2b7e58eb1d [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
21:04:09.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:09.538336 23287 raft_consensus.cc:740] T 00000000000000000000000000000000 P e9dd218ca2244f5a934aac2b7e58eb1d [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: e9dd218ca2244f5a934aac2b7e58eb1d, State: Initialized, Role: FOLLOWER
21:04:09.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:09.539095 23287 consensus_queue.cc:260] T 00000000000000000000000000000000 P e9dd218ca2244f5a934aac2b7e58eb1d [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e9dd218ca2244f5a934aac2b7e58eb1d" member_type: VOTER last_known_addr { host: "127.22.168.62" port: 37239 } } peers { permanent_uuid: "c520077acf3341e29447c6c2b8d3f74c" member_type: VOTER last_known_addr { host: "127.22.168.61" port: 32947 } } peers { permanent_uuid: "eb664add87f44013b596f4cc01c9a41c" member_type: VOTER last_known_addr { host: "127.22.168.60" port: 33247 } }
21:04:09.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:09.540091 23472 sys_catalog.cc:455] T 00000000000000000000000000000000 P e9dd218ca2244f5a934aac2b7e58eb1d [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e9dd218ca2244f5a934aac2b7e58eb1d" member_type: VOTER last_known_addr { host: "127.22.168.62" port: 37239 } } peers { permanent_uuid: "c520077acf3341e29447c6c2b8d3f74c" member_type: VOTER last_known_addr { host: "127.22.168.61" port: 32947 } } peers { permanent_uuid: "eb664add87f44013b596f4cc01c9a41c" member_type: VOTER last_known_addr { host: "127.22.168.60" port: 33247 } } }
21:04:09.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:09.540416 23472 sys_catalog.cc:458] T 00000000000000000000000000000000 P e9dd218ca2244f5a934aac2b7e58eb1d [sys.catalog]: This master's current role is: FOLLOWER
21:04:09.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:09.541464 23287 sys_catalog.cc:565] T 00000000000000000000000000000000 P e9dd218ca2244f5a934aac2b7e58eb1d [sys.catalog]: configured and running, proceeding with master startup.
21:04:09.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:09.541471 23252 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "eb664add87f44013b596f4cc01c9a41c" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "e9dd218ca2244f5a934aac2b7e58eb1d" is_pre_election: true
21:04:09.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:09.541968 23252 raft_consensus.cc:2468] T 00000000000000000000000000000000 P e9dd218ca2244f5a934aac2b7e58eb1d [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate eb664add87f44013b596f4cc01c9a41c in term 0.
21:04:09.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:09.542685 23388 leader_election.cc:304] T 00000000000000000000000000000000 P eb664add87f44013b596f4cc01c9a41c [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: e9dd218ca2244f5a934aac2b7e58eb1d, eb664add87f44013b596f4cc01c9a41c; no voters:
21:04:09.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:09.545029 23458 raft_consensus.cc:2804] T 00000000000000000000000000000000 P eb664add87f44013b596f4cc01c9a41c [term 0 FOLLOWER]: Leader pre-election won for term 1
21:04:09.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:09.545240 23458 raft_consensus.cc:493] T 00000000000000000000000000000000 P eb664add87f44013b596f4cc01c9a41c [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
21:04:09.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:09.545359 23458 raft_consensus.cc:3060] T 00000000000000000000000000000000 P eb664add87f44013b596f4cc01c9a41c [term 0 FOLLOWER]: Advancing to term 1
21:04:09.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:09.549693 23458 raft_consensus.cc:515] T 00000000000000000000000000000000 P eb664add87f44013b596f4cc01c9a41c [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e9dd218ca2244f5a934aac2b7e58eb1d" member_type: VOTER last_known_addr { host: "127.22.168.62" port: 37239 } } peers { permanent_uuid: "c520077acf3341e29447c6c2b8d3f74c" member_type: VOTER last_known_addr { host: "127.22.168.61" port: 32947 } } peers { permanent_uuid: "eb664add87f44013b596f4cc01c9a41c" member_type: VOTER last_known_addr { host: "127.22.168.60" port: 33247 } }
21:04:09.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:09.550726 23458 leader_election.cc:290] T 00000000000000000000000000000000 P eb664add87f44013b596f4cc01c9a41c [CANDIDATE]: Term 1 election: Requested vote from peers e9dd218ca2244f5a934aac2b7e58eb1d (127.22.168.62:37239), c520077acf3341e29447c6c2b8d3f74c (127.22.168.61:32947)
21:04:09.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:09.551638 23252 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "eb664add87f44013b596f4cc01c9a41c" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "e9dd218ca2244f5a934aac2b7e58eb1d"
21:04:09.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:09.551910 23252 raft_consensus.cc:3060] T 00000000000000000000000000000000 P e9dd218ca2244f5a934aac2b7e58eb1d [term 0 FOLLOWER]: Advancing to term 1
21:04:09.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:09.554716 23231 tablet.cc:2378] T 00000000000000000000000000000000 P e9dd218ca2244f5a934aac2b7e58eb1d: Can't schedule compaction. Clean time has not been advanced past its initial value.
21:04:09.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:09.556592 23252 raft_consensus.cc:2468] T 00000000000000000000000000000000 P e9dd218ca2244f5a934aac2b7e58eb1d [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate eb664add87f44013b596f4cc01c9a41c in term 1.
21:04:09.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:09.557265 23388 leader_election.cc:304] T 00000000000000000000000000000000 P eb664add87f44013b596f4cc01c9a41c [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: e9dd218ca2244f5a934aac2b7e58eb1d, eb664add87f44013b596f4cc01c9a41c; no voters:
21:04:09.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:09.557997 23458 raft_consensus.cc:2804] T 00000000000000000000000000000000 P eb664add87f44013b596f4cc01c9a41c [term 1 FOLLOWER]: Leader election won for term 1
21:04:09.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:09.559001 23458 raft_consensus.cc:697] T 00000000000000000000000000000000 P eb664add87f44013b596f4cc01c9a41c [term 1 LEADER]: Becoming Leader. State: Replica: eb664add87f44013b596f4cc01c9a41c, State: Running, Role: LEADER
21:04:09.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:09.559827 23458 consensus_queue.cc:237] T 00000000000000000000000000000000 P eb664add87f44013b596f4cc01c9a41c [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e9dd218ca2244f5a934aac2b7e58eb1d" member_type: VOTER last_known_addr { host: "127.22.168.62" port: 37239 } } peers { permanent_uuid: "c520077acf3341e29447c6c2b8d3f74c" member_type: VOTER last_known_addr { host: "127.22.168.61" port: 32947 } } peers { permanent_uuid: "eb664add87f44013b596f4cc01c9a41c" member_type: VOTER last_known_addr { host: "127.22.168.60" port: 33247 } }
21:04:09.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:09.570570 23477 sys_catalog.cc:455] T 00000000000000000000000000000000 P eb664add87f44013b596f4cc01c9a41c [sys.catalog]: SysCatalogTable state changed. Reason: New leader eb664add87f44013b596f4cc01c9a41c. Latest consensus state: current_term: 1 leader_uuid: "eb664add87f44013b596f4cc01c9a41c" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e9dd218ca2244f5a934aac2b7e58eb1d" member_type: VOTER last_known_addr { host: "127.22.168.62" port: 37239 } } peers { permanent_uuid: "c520077acf3341e29447c6c2b8d3f74c" member_type: VOTER last_known_addr { host: "127.22.168.61" port: 32947 } } peers { permanent_uuid: "eb664add87f44013b596f4cc01c9a41c" member_type: VOTER last_known_addr { host: "127.22.168.60" port: 33247 } } }
21:04:09.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:09.570933 23477 sys_catalog.cc:458] T 00000000000000000000000000000000 P eb664add87f44013b596f4cc01c9a41c [sys.catalog]: This master's current role is: LEADER
21:04:09.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:09.572641 23482 catalog_manager.cc:1485] Loading table and tablet metadata into memory...
21:04:09.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:09.581219 23482 catalog_manager.cc:1494] Initializing Kudu cluster ID...
21:04:09.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:09.583796 23211 master_runner.cc:186] Error getting master registration for 127.22.168.61:32947: OK, instance_id { permanent_uuid: "c520077acf3341e29447c6c2b8d3f74c" instance_seqno: 1761685449001287 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
21:04:09.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:09.595216 23211 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
21:04:09.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:09.602883 23486 catalog_manager.cc:1568] T 00000000000000000000000000000000 P e9dd218ca2244f5a934aac2b7e58eb1d: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
21:04:09.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:09.603555 23486 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
21:04:09.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:09.610729 23252 raft_consensus.cc:1275] T 00000000000000000000000000000000 P e9dd218ca2244f5a934aac2b7e58eb1d [term 1 FOLLOWER]: Refusing update from remote peer eb664add87f44013b596f4cc01c9a41c: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
21:04:09.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:09.611927 23477 consensus_queue.cc:1048] T 00000000000000000000000000000000 P eb664add87f44013b596f4cc01c9a41c [LEADER]: Connected to new peer: Peer: permanent_uuid: "e9dd218ca2244f5a934aac2b7e58eb1d" member_type: VOTER last_known_addr { host: "127.22.168.62" port: 37239 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
21:04:09.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:09.634274 23472 sys_catalog.cc:455] T 00000000000000000000000000000000 P e9dd218ca2244f5a934aac2b7e58eb1d [sys.catalog]: SysCatalogTable state changed. Reason: New leader eb664add87f44013b596f4cc01c9a41c. Latest consensus state: current_term: 1 leader_uuid: "eb664add87f44013b596f4cc01c9a41c" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e9dd218ca2244f5a934aac2b7e58eb1d" member_type: VOTER last_known_addr { host: "127.22.168.62" port: 37239 } } peers { permanent_uuid: "c520077acf3341e29447c6c2b8d3f74c" member_type: VOTER last_known_addr { host: "127.22.168.61" port: 32947 } } peers { permanent_uuid: "eb664add87f44013b596f4cc01c9a41c" member_type: VOTER last_known_addr { host: "127.22.168.60" port: 33247 } } }
21:04:09.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:09.634545 23472 sys_catalog.cc:458] T 00000000000000000000000000000000 P e9dd218ca2244f5a934aac2b7e58eb1d [sys.catalog]: This master's current role is: FOLLOWER
21:04:09.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:09.638335 23488 mvcc.cc:204] Tried to move back new op lower bound from 7215863601590702080 to 7215863601404362752. Current Snapshot: MvccSnapshot[applied={T|T < 7215863601590702080}]
21:04:09.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:09.643252 23472 sys_catalog.cc:455] T 00000000000000000000000000000000 P e9dd218ca2244f5a934aac2b7e58eb1d [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "eb664add87f44013b596f4cc01c9a41c" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e9dd218ca2244f5a934aac2b7e58eb1d" member_type: VOTER last_known_addr { host: "127.22.168.62" port: 37239 } } peers { permanent_uuid: "c520077acf3341e29447c6c2b8d3f74c" member_type: VOTER last_known_addr { host: "127.22.168.61" port: 32947 } } peers { permanent_uuid: "eb664add87f44013b596f4cc01c9a41c" member_type: VOTER last_known_addr { host: "127.22.168.60" port: 33247 } } }
21:04:09.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:09.643487 23472 sys_catalog.cc:458] T 00000000000000000000000000000000 P e9dd218ca2244f5a934aac2b7e58eb1d [sys.catalog]: This master's current role is: FOLLOWER
21:04:09.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:09.645495 23489 mvcc.cc:204] Tried to move back new op lower bound from 7215863601590702080 to 7215863601404362752. Current Snapshot: MvccSnapshot[applied={T|T < 7215863601590702080}]
21:04:09.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:09.648949 23458 sys_catalog.cc:455] T 00000000000000000000000000000000 P eb664add87f44013b596f4cc01c9a41c [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "eb664add87f44013b596f4cc01c9a41c" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e9dd218ca2244f5a934aac2b7e58eb1d" member_type: VOTER last_known_addr { host: "127.22.168.62" port: 37239 } } peers { permanent_uuid: "c520077acf3341e29447c6c2b8d3f74c" member_type: VOTER last_known_addr { host: "127.22.168.61" port: 32947 } } peers { permanent_uuid: "eb664add87f44013b596f4cc01c9a41c" member_type: VOTER last_known_addr { host: "127.22.168.60" port: 33247 } } }
21:04:09.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:09.649250 23458 sys_catalog.cc:458] T 00000000000000000000000000000000 P eb664add87f44013b596f4cc01c9a41c [sys.catalog]: This master's current role is: LEADER
21:04:09.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:09.654695 23482 catalog_manager.cc:1357] Generated new cluster ID: 43f2ae537b96433b8f15997b1c5c1cd1
21:04:09.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:09.654827 23482 catalog_manager.cc:1505] Initializing Kudu internal certificate authority...
21:04:09.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:09.673520 23482 catalog_manager.cc:1380] Generated new certificate authority record
21:04:09.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:09.677194 23482 catalog_manager.cc:1514] Loading token signing keys...
21:04:09.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:09.702109 23368 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.22.168.60" port: 33247 } attempt: 5
21:04:09.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:09.703861 23482 catalog_manager.cc:6022] T 00000000000000000000000000000000 P eb664add87f44013b596f4cc01c9a41c: Generated new TSK 0
21:04:09.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:09.706460 23482 catalog_manager.cc:1524] Initializing in-progress tserver states...
21:04:09.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:09.716632 23200 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskRxIrUU/build/asan/bin/kudu
21:04:09.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskRxIrUU/build/asan/bin/kudu
21:04:09.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskRxIrUU/test-tmp/mini-kudu-cluster8143392791847762722/ts-0/wal
21:04:09.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskRxIrUU/test-tmp/mini-kudu-cluster8143392791847762722/ts-0/data
21:04:09.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
21:04:09.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
21:04:09.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
21:04:09.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:04:09.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:04:09.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:04:09.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskRxIrUU/test-tmp/mini-kudu-cluster8143392791847762722/ts-0/logs
21:04:09.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskRxIrUU/test-tmp/mini-kudu-cluster8143392791847762722/ts-0/data/info.pb
21:04:09.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:04:09.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
21:04:09.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
21:04:09.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
21:04:09.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
21:04:09.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:04:09.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:04:09.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:04:09.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
21:04:09.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
21:04:09.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.22.168.1:0
21:04:09.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.22.168.1
21:04:09.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.22.168.1
21:04:09.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:04:09.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.22.168.62:37239,127.22.168.61:32947,127.22.168.60:33247
21:04:09.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.22.168.20:37293
21:04:09.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:04:09.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:04:09.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:04:09.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_keepalive_interval_ms=1000
21:04:09.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_staleness_tracker_interval_ms=250
21:04:09.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_txn_system_client_init=true with env {}
21:04:09.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:09.720201 23368 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P c520077acf3341e29447c6c2b8d3f74c: Bootstrap starting.
21:04:09.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:09.729512 23368 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P c520077acf3341e29447c6c2b8d3f74c: Neither blocks nor log segments found. Creating new log.
21:04:09.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:09.732949 23368 log.cc:826] T 00000000000000000000000000000000 P c520077acf3341e29447c6c2b8d3f74c: Log is configured to *not* fsync() on all Append() calls
21:04:09.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:09.754233 23368 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P c520077acf3341e29447c6c2b8d3f74c: No bootstrap required, opened a new log
21:04:09.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:09.771456 23368 raft_consensus.cc:359] T 00000000000000000000000000000000 P c520077acf3341e29447c6c2b8d3f74c [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e9dd218ca2244f5a934aac2b7e58eb1d" member_type: VOTER last_known_addr { host: "127.22.168.62" port: 37239 } } peers { permanent_uuid: "c520077acf3341e29447c6c2b8d3f74c" member_type: VOTER last_known_addr { host: "127.22.168.61" port: 32947 } } peers { permanent_uuid: "eb664add87f44013b596f4cc01c9a41c" member_type: VOTER last_known_addr { host: "127.22.168.60" port: 33247 } }
21:04:09.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:09.772217 23368 raft_consensus.cc:385] T 00000000000000000000000000000000 P c520077acf3341e29447c6c2b8d3f74c [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
21:04:09.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:09.772553 23368 raft_consensus.cc:740] T 00000000000000000000000000000000 P c520077acf3341e29447c6c2b8d3f74c [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: c520077acf3341e29447c6c2b8d3f74c, State: Initialized, Role: FOLLOWER
21:04:09.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:09.774071 23368 consensus_queue.cc:260] T 00000000000000000000000000000000 P c520077acf3341e29447c6c2b8d3f74c [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e9dd218ca2244f5a934aac2b7e58eb1d" member_type: VOTER last_known_addr { host: "127.22.168.62" port: 37239 } } peers { permanent_uuid: "c520077acf3341e29447c6c2b8d3f74c" member_type: VOTER last_known_addr { host: "127.22.168.61" port: 32947 } } peers { permanent_uuid: "eb664add87f44013b596f4cc01c9a41c" member_type: VOTER last_known_addr { host: "127.22.168.60" port: 33247 } }
21:04:09.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:09.788307 23332 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "eb664add87f44013b596f4cc01c9a41c" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c520077acf3341e29447c6c2b8d3f74c"
21:04:09.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:09.788865 23332 raft_consensus.cc:3060] T 00000000000000000000000000000000 P c520077acf3341e29447c6c2b8d3f74c [term 0 FOLLOWER]: Advancing to term 1
21:04:09.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:09.792729 23333 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "eb664add87f44013b596f4cc01c9a41c" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c520077acf3341e29447c6c2b8d3f74c" is_pre_election: true
21:04:09.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:09.792416 23332 raft_consensus.cc:2468] T 00000000000000000000000000000000 P c520077acf3341e29447c6c2b8d3f74c [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate eb664add87f44013b596f4cc01c9a41c in term 1.
21:04:09.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:09.795601 23331 raft_consensus.cc:1275] T 00000000000000000000000000000000 P c520077acf3341e29447c6c2b8d3f74c [term 1 FOLLOWER]: Refusing update from remote peer eb664add87f44013b596f4cc01c9a41c: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
21:04:09.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:09.797863 23458 consensus_queue.cc:1048] T 00000000000000000000000000000000 P eb664add87f44013b596f4cc01c9a41c [LEADER]: Connected to new peer: Peer: permanent_uuid: "c520077acf3341e29447c6c2b8d3f74c" member_type: VOTER last_known_addr { host: "127.22.168.61" port: 32947 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
21:04:09.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:09.800776 23332 rpcz_store.cc:267] Call kudu.consensus.ConsensusService.RequestConsensusVote from 127.0.0.1:36992 (request call id 2) took 249 ms (client timeout 305 ms). Trace:
21:04:09.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:09.801110 23332 rpcz_store.cc:269] 1028 21:04:09.551069 (+ 0us) service_pool.cc:168] Inserting onto call queue
21:04:09.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1028 21:04:09.551141 (+ 72us) service_pool.cc:225] Handling call
21:04:09.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1028 21:04:09.800721 (+249580us) inbound_call.cc:173] Queueing success response
21:04:09.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {}
21:04:09.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:09.805444 23368 sys_catalog.cc:565] T 00000000000000000000000000000000 P c520077acf3341e29447c6c2b8d3f74c [sys.catalog]: configured and running, proceeding with master startup.
21:04:09.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:09.853374 23497 sys_catalog.cc:455] T 00000000000000000000000000000000 P c520077acf3341e29447c6c2b8d3f74c [sys.catalog]: SysCatalogTable state changed. Reason: New leader eb664add87f44013b596f4cc01c9a41c. Latest consensus state: current_term: 1 leader_uuid: "eb664add87f44013b596f4cc01c9a41c" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e9dd218ca2244f5a934aac2b7e58eb1d" member_type: VOTER last_known_addr { host: "127.22.168.62" port: 37239 } } peers { permanent_uuid: "c520077acf3341e29447c6c2b8d3f74c" member_type: VOTER last_known_addr { host: "127.22.168.61" port: 32947 } } peers { permanent_uuid: "eb664add87f44013b596f4cc01c9a41c" member_type: VOTER last_known_addr { host: "127.22.168.60" port: 33247 } } }
21:04:09.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:09.853708 23497 sys_catalog.cc:458] T 00000000000000000000000000000000 P c520077acf3341e29447c6c2b8d3f74c [sys.catalog]: This master's current role is: FOLLOWER
21:04:09.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:09.854105 23496 sys_catalog.cc:455] T 00000000000000000000000000000000 P c520077acf3341e29447c6c2b8d3f74c [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 1 leader_uuid: "eb664add87f44013b596f4cc01c9a41c" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e9dd218ca2244f5a934aac2b7e58eb1d" member_type: VOTER last_known_addr { host: "127.22.168.62" port: 37239 } } peers { permanent_uuid: "c520077acf3341e29447c6c2b8d3f74c" member_type: VOTER last_known_addr { host: "127.22.168.61" port: 32947 } } peers { permanent_uuid: "eb664add87f44013b596f4cc01c9a41c" member_type: VOTER last_known_addr { host: "127.22.168.60" port: 33247 } } }
21:04:09.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:09.854367 23496 sys_catalog.cc:458] T 00000000000000000000000000000000 P c520077acf3341e29447c6c2b8d3f74c [sys.catalog]: This master's current role is: FOLLOWER
21:04:09.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:09.854990 23503 sys_catalog.cc:455] T 00000000000000000000000000000000 P c520077acf3341e29447c6c2b8d3f74c [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "eb664add87f44013b596f4cc01c9a41c" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e9dd218ca2244f5a934aac2b7e58eb1d" member_type: VOTER last_known_addr { host: "127.22.168.62" port: 37239 } } peers { permanent_uuid: "c520077acf3341e29447c6c2b8d3f74c" member_type: VOTER last_known_addr { host: "127.22.168.61" port: 32947 } } peers { permanent_uuid: "eb664add87f44013b596f4cc01c9a41c" member_type: VOTER last_known_addr { host: "127.22.168.60" port: 33247 } } }
21:04:09.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:09.855427 23503 sys_catalog.cc:458] T 00000000000000000000000000000000 P c520077acf3341e29447c6c2b8d3f74c [sys.catalog]: This master's current role is: FOLLOWER
21:04:09.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:09.886390 23514 catalog_manager.cc:805] Waiting for catalog manager background task thread to start: Service unavailable: Catalog manager is not initialized. State: Starting
21:04:09.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:09.895304 23493 sys_catalog.cc:455] T 00000000000000000000000000000000 P eb664add87f44013b596f4cc01c9a41c [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "eb664add87f44013b596f4cc01c9a41c" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e9dd218ca2244f5a934aac2b7e58eb1d" member_type: VOTER last_known_addr { host: "127.22.168.62" port: 37239 } } peers { permanent_uuid: "c520077acf3341e29447c6c2b8d3f74c" member_type: VOTER last_known_addr { host: "127.22.168.61" port: 32947 } } peers { permanent_uuid: "eb664add87f44013b596f4cc01c9a41c" member_type: VOTER last_known_addr { host: "127.22.168.60" port: 33247 } } }
21:04:09.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:09.895604 23493 sys_catalog.cc:458] T 00000000000000000000000000000000 P eb664add87f44013b596f4cc01c9a41c [sys.catalog]: This master's current role is: LEADER
21:04:10.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:10.189801 23494 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
21:04:10.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:10.190126 23494 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
21:04:10.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:10.190229 23494 flags.cc:425] Enabled unsafe flag: --never_fsync=true
21:04:10.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:10.198535 23494 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
21:04:10.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:10.198679 23494 flags.cc:425] Enabled experimental flag: --txn_keepalive_interval_ms=1000
21:04:10.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:10.198730 23494 flags.cc:425] Enabled experimental flag: --txn_staleness_tracker_interval_ms=250
21:04:10.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:10.198766 23494 flags.cc:425] Enabled experimental flag: --enable_txn_system_client_init=true
21:04:10.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:10.198845 23494 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.22.168.1
21:04:10.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:10.209118 23494 tablet_server_runner.cc:78] Tablet server non-default flags:
21:04:10.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:04:10.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.22.168.20:37293
21:04:10.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:04:10.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:04:10.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskRxIrUU/test-tmp/mini-kudu-cluster8143392791847762722/ts-0/data
21:04:10.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskRxIrUU/test-tmp/mini-kudu-cluster8143392791847762722/ts-0/wal
21:04:10.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:04:10.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:04:10.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.22.168.1:0
21:04:10.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
21:04:10.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:04:10.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:04:10.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskRxIrUU/test-tmp/mini-kudu-cluster8143392791847762722/ts-0/data/info.pb
21:04:10.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.22.168.1
21:04:10.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:04:10.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_keepalive_interval_ms=1000
21:04:10.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_staleness_tracker_interval_ms=250
21:04:10.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_txn_system_client_init=true
21:04:10.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.22.168.62:37239,127.22.168.61:32947,127.22.168.60:33247
21:04:10.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
21:04:10.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:04:10.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
21:04:10.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
21:04:10.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:04:10.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.22.168.1
21:04:10.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskRxIrUU/test-tmp/mini-kudu-cluster8143392791847762722/ts-0/logs
21:04:10.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:04:10.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
21:04:10.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
21:04:10.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
21:04:10.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
21:04:10.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 60f5e5267b92c39485a66121d3ce3cc7ef57b0e0
21:04:10.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
21:04:10.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 28 Oct 2025 20:43:24 UTC on 24a791456cd2
21:04:10.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 8764
21:04:10.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
21:04:10.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:10.210753 23494 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
21:04:10.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:10.212714 23494 file_cache.cc:492] Constructed file cache file cache with capacity 419430
21:04:10.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:10.222345 23519 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:04:10.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:10.222369 23520 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:04:10.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:10.223140 23522 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:04:10.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:10.223649 23494 server_base.cc:1047] running on GCE node
21:04:10.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:10.224272 23494 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
21:04:10.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:10.225224 23494 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
21:04:10.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:10.226464 23494 hybrid_clock.cc:648] HybridClock initialized: now 1761685450226406 us; error 27 us; skew 500 ppm
21:04:10.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:10.226789 23494 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
21:04:10.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:10.229128 23494 webserver.cc:492] Webserver started at http://127.22.168.1:43979/ using document root <none> and password file <none>
21:04:10.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:10.229861 23494 fs_manager.cc:362] Metadata directory not provided
21:04:10.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:10.229993 23494 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
21:04:10.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:10.230356 23494 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
21:04:10.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:10.232985 23494 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskRxIrUU/test-tmp/mini-kudu-cluster8143392791847762722/ts-0/data/instance:
21:04:10.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "210cdae6f0e44d83acf5641ce563b738"
21:04:10.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-28 21:04:10 on dist-test-slave-vx1b"
21:04:10.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:10.233752 23494 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskRxIrUU/test-tmp/mini-kudu-cluster8143392791847762722/ts-0/wal/instance:
21:04:10.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "210cdae6f0e44d83acf5641ce563b738"
21:04:10.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-28 21:04:10 on dist-test-slave-vx1b"
21:04:10.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:10.238972 23494 fs_manager.cc:696] Time spent creating directory manager: real 0.005s user 0.005s sys 0.000s
21:04:10.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:10.242249 23528 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
21:04:10.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:10.243618 23494 fs_manager.cc:730] Time spent opening block manager: real 0.003s user 0.002s sys 0.000s
21:04:10.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:10.243777 23494 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskRxIrUU/test-tmp/mini-kudu-cluster8143392791847762722/ts-0/data,/tmp/dist-test-taskRxIrUU/test-tmp/mini-kudu-cluster8143392791847762722/ts-0/wal
21:04:10.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "210cdae6f0e44d83acf5641ce563b738"
21:04:10.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-28 21:04:10 on dist-test-slave-vx1b"
21:04:10.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:10.243999 23494 fs_report.cc:389] FS layout report
21:04:10.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
21:04:10.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskRxIrUU/test-tmp/mini-kudu-cluster8143392791847762722/ts-0/wal
21:04:10.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskRxIrUU/test-tmp/mini-kudu-cluster8143392791847762722/ts-0/wal
21:04:10.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskRxIrUU/test-tmp/mini-kudu-cluster8143392791847762722/ts-0/data/data
21:04:10.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
21:04:10.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
21:04:10.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
21:04:10.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
21:04:10.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
21:04:10.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
21:04:10.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
21:04:10.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
21:04:10.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
21:04:10.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
21:04:10.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
21:04:10.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:10.281145 23494 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
21:04:10.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:10.282048 23494 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
21:04:10.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:10.282358 23494 kserver.cc:163] Server-wide thread pool size limit: 3276
21:04:10.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:10.285781 23494 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
21:04:10.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:10.286048 23494 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s user 0.001s sys 0.000s
21:04:10.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:10.286285 23494 ts_tablet_manager.cc:616] Registered 0 tablets
21:04:10.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:10.286378 23494 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
21:04:10.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:10.348229 23494 rpc_server.cc:307] RPC server started. Bound to: 127.22.168.1:42241
21:04:10.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:10.348297 23652 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.22.168.1:42241 every 8 connection(s)
21:04:10.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:10.349704 23494 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskRxIrUU/test-tmp/mini-kudu-cluster8143392791847762722/ts-0/data/info.pb
21:04:10.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:10.354779 23654 heartbeater.cc:344] Connected to a master server at 127.22.168.62:37239
21:04:10.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:10.356295 23654 heartbeater.cc:461] Registering TS with master...
21:04:10.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:10.357996 23200 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskRxIrUU/build/asan/bin/kudu as pid 23494
21:04:10.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:10.358197 23200 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskRxIrUU/test-tmp/mini-kudu-cluster8143392791847762722/ts-0/wal/instance
21:04:10.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:10.359958 23654 heartbeater.cc:507] Master 127.22.168.62:37239 requested a full tablet report, sending...
21:04:10.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:10.362046 23653 heartbeater.cc:344] Connected to a master server at 127.22.168.60:33247
21:04:10.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:10.362294 23653 heartbeater.cc:461] Registering TS with master...
21:04:10.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:10.362913 23653 heartbeater.cc:507] Master 127.22.168.60:33247 requested a full tablet report, sending...
21:04:10.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:10.363070 23242 ts_manager.cc:194] Registered new tserver with Master: 210cdae6f0e44d83acf5641ce563b738 (127.22.168.1:42241)
21:04:10.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:10.363143 23200 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskRxIrUU/build/asan/bin/kudu
21:04:10.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskRxIrUU/build/asan/bin/kudu
21:04:10.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskRxIrUU/test-tmp/mini-kudu-cluster8143392791847762722/ts-1/wal
21:04:10.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskRxIrUU/test-tmp/mini-kudu-cluster8143392791847762722/ts-1/data
21:04:10.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
21:04:10.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
21:04:10.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
21:04:10.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:04:10.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:04:10.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:04:10.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskRxIrUU/test-tmp/mini-kudu-cluster8143392791847762722/ts-1/logs
21:04:10.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskRxIrUU/test-tmp/mini-kudu-cluster8143392791847762722/ts-1/data/info.pb
21:04:10.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:04:10.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
21:04:10.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
21:04:10.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
21:04:10.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
21:04:10.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:04:10.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:04:10.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:04:10.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
21:04:10.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
21:04:10.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.22.168.2:0
21:04:10.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.22.168.2
21:04:10.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.22.168.2
21:04:10.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:04:10.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.22.168.62:37239,127.22.168.61:32947,127.22.168.60:33247
21:04:10.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.22.168.20:37293
21:04:10.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:04:10.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:04:10.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:04:10.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_keepalive_interval_ms=1000
21:04:10.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_staleness_tracker_interval_ms=250
21:04:10.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_txn_system_client_init=true with env {}
21:04:10.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:10.364514 23655 heartbeater.cc:344] Connected to a master server at 127.22.168.61:32947
21:04:10.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:10.364683 23655 heartbeater.cc:461] Registering TS with master...
21:04:10.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:10.365121 23655 heartbeater.cc:507] Master 127.22.168.61:32947 requested a full tablet report, sending...
21:04:10.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:10.365211 23404 ts_manager.cc:194] Registered new tserver with Master: 210cdae6f0e44d83acf5641ce563b738 (127.22.168.1:42241)
21:04:10.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:10.366712 23323 ts_manager.cc:194] Registered new tserver with Master: 210cdae6f0e44d83acf5641ce563b738 (127.22.168.1:42241)
21:04:10.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:10.367244 23404 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.22.168.1:56411
21:04:10.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:10.606204 23486 catalog_manager.cc:1269] Loaded cluster ID: 43f2ae537b96433b8f15997b1c5c1cd1
21:04:10.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:10.606323 23486 catalog_manager.cc:1562] T 00000000000000000000000000000000 P e9dd218ca2244f5a934aac2b7e58eb1d: loading cluster ID for follower catalog manager: success
21:04:10.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:10.610332 23486 catalog_manager.cc:1584] T 00000000000000000000000000000000 P e9dd218ca2244f5a934aac2b7e58eb1d: acquiring CA information for follower catalog manager: success
21:04:10.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:10.612896 23486 catalog_manager.cc:1612] T 00000000000000000000000000000000 P e9dd218ca2244f5a934aac2b7e58eb1d: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
21:04:10.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:10.688357 23658 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
21:04:10.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:10.688743 23658 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
21:04:10.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:10.688898 23658 flags.cc:425] Enabled unsafe flag: --never_fsync=true
21:04:10.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:10.697312 23658 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
21:04:10.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:10.697486 23658 flags.cc:425] Enabled experimental flag: --txn_keepalive_interval_ms=1000
21:04:10.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:10.697557 23658 flags.cc:425] Enabled experimental flag: --txn_staleness_tracker_interval_ms=250
21:04:10.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:10.697619 23658 flags.cc:425] Enabled experimental flag: --enable_txn_system_client_init=true
21:04:10.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:10.697749 23658 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.22.168.2
21:04:10.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:10.707571 23658 tablet_server_runner.cc:78] Tablet server non-default flags:
21:04:10.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:04:10.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.22.168.20:37293
21:04:10.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:04:10.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:04:10.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskRxIrUU/test-tmp/mini-kudu-cluster8143392791847762722/ts-1/data
21:04:10.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskRxIrUU/test-tmp/mini-kudu-cluster8143392791847762722/ts-1/wal
21:04:10.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:04:10.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:04:10.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.22.168.2:0
21:04:10.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
21:04:10.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:04:10.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:04:10.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskRxIrUU/test-tmp/mini-kudu-cluster8143392791847762722/ts-1/data/info.pb
21:04:10.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.22.168.2
21:04:10.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:04:10.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_keepalive_interval_ms=1000
21:04:10.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_staleness_tracker_interval_ms=250
21:04:10.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_txn_system_client_init=true
21:04:10.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.22.168.62:37239,127.22.168.61:32947,127.22.168.60:33247
21:04:10.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
21:04:10.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:04:10.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
21:04:10.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
21:04:10.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:04:10.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.22.168.2
21:04:10.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskRxIrUU/test-tmp/mini-kudu-cluster8143392791847762722/ts-1/logs
21:04:10.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:04:10.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
21:04:10.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
21:04:10.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
21:04:10.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
21:04:10.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 60f5e5267b92c39485a66121d3ce3cc7ef57b0e0
21:04:10.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
21:04:10.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 28 Oct 2025 20:43:24 UTC on 24a791456cd2
21:04:10.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 8764
21:04:10.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
21:04:10.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:10.709600 23658 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
21:04:10.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:10.711335 23658 file_cache.cc:492] Constructed file cache file cache with capacity 419430
21:04:10.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:10.721204 23663 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:04:10.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:10.721251 23664 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:04:10.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:10.721491 23666 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:04:10.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:10.722138 23658 server_base.cc:1047] running on GCE node
21:04:10.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:10.722694 23658 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
21:04:10.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:10.723567 23658 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
21:04:10.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:10.724812 23658 hybrid_clock.cc:648] HybridClock initialized: now 1761685450724747 us; error 37 us; skew 500 ppm
21:04:10.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:10.725129 23658 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
21:04:10.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:10.727365 23658 webserver.cc:492] Webserver started at http://127.22.168.2:39545/ using document root <none> and password file <none>
21:04:10.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:10.728113 23658 fs_manager.cc:362] Metadata directory not provided
21:04:10.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:10.728237 23658 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
21:04:10.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:10.728605 23658 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
21:04:10.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:10.731026 23658 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskRxIrUU/test-tmp/mini-kudu-cluster8143392791847762722/ts-1/data/instance:
21:04:10.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "1acec06b763242d082ddb75231fc4eca"
21:04:10.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-28 21:04:10 on dist-test-slave-vx1b"
21:04:10.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:10.731782 23658 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskRxIrUU/test-tmp/mini-kudu-cluster8143392791847762722/ts-1/wal/instance:
21:04:10.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "1acec06b763242d082ddb75231fc4eca"
21:04:10.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-28 21:04:10 on dist-test-slave-vx1b"
21:04:10.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:10.737119 23658 fs_manager.cc:696] Time spent creating directory manager: real 0.005s user 0.005s sys 0.002s
21:04:10.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:10.740694 23672 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
21:04:10.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:10.742210 23658 fs_manager.cc:730] Time spent opening block manager: real 0.003s user 0.004s sys 0.000s
21:04:10.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:10.742381 23658 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskRxIrUU/test-tmp/mini-kudu-cluster8143392791847762722/ts-1/data,/tmp/dist-test-taskRxIrUU/test-tmp/mini-kudu-cluster8143392791847762722/ts-1/wal
21:04:10.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "1acec06b763242d082ddb75231fc4eca"
21:04:10.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-28 21:04:10 on dist-test-slave-vx1b"
21:04:10.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:10.742587 23658 fs_report.cc:389] FS layout report
21:04:10.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
21:04:10.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskRxIrUU/test-tmp/mini-kudu-cluster8143392791847762722/ts-1/wal
21:04:10.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskRxIrUU/test-tmp/mini-kudu-cluster8143392791847762722/ts-1/wal
21:04:10.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskRxIrUU/test-tmp/mini-kudu-cluster8143392791847762722/ts-1/data/data
21:04:10.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
21:04:10.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
21:04:10.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
21:04:10.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
21:04:10.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
21:04:10.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
21:04:10.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
21:04:10.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
21:04:10.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
21:04:10.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
21:04:10.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
21:04:10.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:10.758041 23658 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
21:04:10.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:10.758874 23658 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
21:04:10.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:10.759182 23658 kserver.cc:163] Server-wide thread pool size limit: 3276
21:04:10.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:10.762308 23658 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
21:04:10.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:10.762421 23658 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
21:04:10.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:10.762533 23658 ts_tablet_manager.cc:616] Registered 0 tablets
21:04:10.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:10.762601 23658 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
21:04:10.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:10.821928 23658 rpc_server.cc:307] RPC server started. Bound to: 127.22.168.2:37409
21:04:10.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:10.821990 23793 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.22.168.2:37409 every 8 connection(s)
21:04:10.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:10.823328 23658 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskRxIrUU/test-tmp/mini-kudu-cluster8143392791847762722/ts-1/data/info.pb
21:04:10.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:10.823643 23200 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskRxIrUU/build/asan/bin/kudu as pid 23658
21:04:10.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:10.823810 23200 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskRxIrUU/test-tmp/mini-kudu-cluster8143392791847762722/ts-1/wal/instance
21:04:10.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:10.828125 23795 heartbeater.cc:344] Connected to a master server at 127.22.168.62:37239
21:04:10.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:10.828711 23200 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskRxIrUU/build/asan/bin/kudu
21:04:10.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskRxIrUU/build/asan/bin/kudu
21:04:10.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskRxIrUU/test-tmp/mini-kudu-cluster8143392791847762722/ts-2/wal
21:04:10.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskRxIrUU/test-tmp/mini-kudu-cluster8143392791847762722/ts-2/data
21:04:10.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
21:04:10.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
21:04:10.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
21:04:10.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:04:10.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:04:10.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:04:10.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskRxIrUU/test-tmp/mini-kudu-cluster8143392791847762722/ts-2/logs
21:04:10.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskRxIrUU/test-tmp/mini-kudu-cluster8143392791847762722/ts-2/data/info.pb
21:04:10.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:04:10.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
21:04:10.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
21:04:10.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
21:04:10.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
21:04:10.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:04:10.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:04:10.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:04:10.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
21:04:10.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
21:04:10.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.22.168.3:0
21:04:10.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.22.168.3
21:04:10.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.22.168.3
21:04:10.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:04:10.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.22.168.62:37239,127.22.168.61:32947,127.22.168.60:33247
21:04:10.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.22.168.20:37293
21:04:10.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:04:10.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:04:10.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:04:10.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_keepalive_interval_ms=1000
21:04:10.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_staleness_tracker_interval_ms=250
21:04:10.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_txn_system_client_init=true with env {}
21:04:10.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:10.828758 23795 heartbeater.cc:461] Registering TS with master...
21:04:10.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:10.830503 23795 heartbeater.cc:507] Master 127.22.168.62:37239 requested a full tablet report, sending...
21:04:10.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:10.832528 23242 ts_manager.cc:194] Registered new tserver with Master: 1acec06b763242d082ddb75231fc4eca (127.22.168.2:37409)
21:04:10.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:10.835968 23794 heartbeater.cc:344] Connected to a master server at 127.22.168.60:33247
21:04:10.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:10.836308 23794 heartbeater.cc:461] Registering TS with master...
21:04:10.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:10.836999 23794 heartbeater.cc:507] Master 127.22.168.60:33247 requested a full tablet report, sending...
21:04:10.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:10.838095 23404 ts_manager.cc:194] Registered new tserver with Master: 1acec06b763242d082ddb75231fc4eca (127.22.168.2:37409)
21:04:10.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:10.839226 23404 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.22.168.2:40155
21:04:10.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:10.841833 23796 heartbeater.cc:344] Connected to a master server at 127.22.168.61:32947
21:04:10.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:10.841988 23796 heartbeater.cc:461] Registering TS with master...
21:04:10.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:10.842356 23796 heartbeater.cc:507] Master 127.22.168.61:32947 requested a full tablet report, sending...
21:04:10.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:10.843302 23323 ts_manager.cc:194] Registered new tserver with Master: 1acec06b763242d082ddb75231fc4eca (127.22.168.2:37409)
21:04:10.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:10.891637 23514 catalog_manager.cc:1269] Loaded cluster ID: 43f2ae537b96433b8f15997b1c5c1cd1
21:04:10.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:10.891783 23514 catalog_manager.cc:1562] T 00000000000000000000000000000000 P c520077acf3341e29447c6c2b8d3f74c: loading cluster ID for follower catalog manager: success
21:04:10.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:10.895774 23514 catalog_manager.cc:1584] T 00000000000000000000000000000000 P c520077acf3341e29447c6c2b8d3f74c: acquiring CA information for follower catalog manager: success
21:04:10.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:10.898335 23514 catalog_manager.cc:1612] T 00000000000000000000000000000000 P c520077acf3341e29447c6c2b8d3f74c: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
21:04:11.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:11.169481 23798 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
21:04:11.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:11.169816 23798 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
21:04:11.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:11.169960 23798 flags.cc:425] Enabled unsafe flag: --never_fsync=true
21:04:11.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:11.178097 23798 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
21:04:11.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:11.178270 23798 flags.cc:425] Enabled experimental flag: --txn_keepalive_interval_ms=1000
21:04:11.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:11.178342 23798 flags.cc:425] Enabled experimental flag: --txn_staleness_tracker_interval_ms=250
21:04:11.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:11.178396 23798 flags.cc:425] Enabled experimental flag: --enable_txn_system_client_init=true
21:04:11.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:11.178524 23798 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.22.168.3
21:04:11.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:11.188313 23798 tablet_server_runner.cc:78] Tablet server non-default flags:
21:04:11.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:04:11.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.22.168.20:37293
21:04:11.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:04:11.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:04:11.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskRxIrUU/test-tmp/mini-kudu-cluster8143392791847762722/ts-2/data
21:04:11.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskRxIrUU/test-tmp/mini-kudu-cluster8143392791847762722/ts-2/wal
21:04:11.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:04:11.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:04:11.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.22.168.3:0
21:04:11.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
21:04:11.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:04:11.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:04:11.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskRxIrUU/test-tmp/mini-kudu-cluster8143392791847762722/ts-2/data/info.pb
21:04:11.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.22.168.3
21:04:11.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:04:11.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_keepalive_interval_ms=1000
21:04:11.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_staleness_tracker_interval_ms=250
21:04:11.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_txn_system_client_init=true
21:04:11.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.22.168.62:37239,127.22.168.61:32947,127.22.168.60:33247
21:04:11.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
21:04:11.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:04:11.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
21:04:11.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
21:04:11.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:04:11.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.22.168.3
21:04:11.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskRxIrUU/test-tmp/mini-kudu-cluster8143392791847762722/ts-2/logs
21:04:11.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:04:11.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
21:04:11.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
21:04:11.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
21:04:11.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
21:04:11.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 60f5e5267b92c39485a66121d3ce3cc7ef57b0e0
21:04:11.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
21:04:11.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 28 Oct 2025 20:43:24 UTC on 24a791456cd2
21:04:11.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 8764
21:04:11.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
21:04:11.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:11.190207 23798 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
21:04:11.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:11.191964 23798 file_cache.cc:492] Constructed file cache file cache with capacity 419430
21:04:11.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:11.201195 23805 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:04:11.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:11.202364 23806 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:04:11.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:11.203821 23808 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:04:11.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:11.204485 23798 server_base.cc:1047] running on GCE node
21:04:11.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:11.205036 23798 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
21:04:11.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:11.205999 23798 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
21:04:11.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:11.207228 23798 hybrid_clock.cc:648] HybridClock initialized: now 1761685451207145 us; error 59 us; skew 500 ppm
21:04:11.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:11.207525 23798 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
21:04:11.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:11.209932 23798 webserver.cc:492] Webserver started at http://127.22.168.3:32991/ using document root <none> and password file <none>
21:04:11.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:11.210687 23798 fs_manager.cc:362] Metadata directory not provided
21:04:11.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:11.210790 23798 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
21:04:11.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:11.211144 23798 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
21:04:11.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:11.214030 23798 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskRxIrUU/test-tmp/mini-kudu-cluster8143392791847762722/ts-2/data/instance:
21:04:11.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "e24858bd4ab14759b2252ddad7f0e5cd"
21:04:11.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-28 21:04:11 on dist-test-slave-vx1b"
21:04:11.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:11.214883 23798 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskRxIrUU/test-tmp/mini-kudu-cluster8143392791847762722/ts-2/wal/instance:
21:04:11.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "e24858bd4ab14759b2252ddad7f0e5cd"
21:04:11.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-28 21:04:11 on dist-test-slave-vx1b"
21:04:11.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:11.220216 23798 fs_manager.cc:696] Time spent creating directory manager: real 0.005s user 0.006s sys 0.000s
21:04:11.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:11.223888 23814 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
21:04:11.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:11.225244 23798 fs_manager.cc:730] Time spent opening block manager: real 0.003s user 0.003s sys 0.000s
21:04:11.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:11.225379 23798 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskRxIrUU/test-tmp/mini-kudu-cluster8143392791847762722/ts-2/data,/tmp/dist-test-taskRxIrUU/test-tmp/mini-kudu-cluster8143392791847762722/ts-2/wal
21:04:11.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "e24858bd4ab14759b2252ddad7f0e5cd"
21:04:11.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-28 21:04:11 on dist-test-slave-vx1b"
21:04:11.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:11.225538 23798 fs_report.cc:389] FS layout report
21:04:11.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
21:04:11.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskRxIrUU/test-tmp/mini-kudu-cluster8143392791847762722/ts-2/wal
21:04:11.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskRxIrUU/test-tmp/mini-kudu-cluster8143392791847762722/ts-2/wal
21:04:11.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskRxIrUU/test-tmp/mini-kudu-cluster8143392791847762722/ts-2/data/data
21:04:11.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
21:04:11.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
21:04:11.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
21:04:11.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
21:04:11.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
21:04:11.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
21:04:11.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
21:04:11.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
21:04:11.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
21:04:11.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
21:04:11.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
21:04:11.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:11.238905 23798 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
21:04:11.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:11.239751 23798 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
21:04:11.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:11.240072 23798 kserver.cc:163] Server-wide thread pool size limit: 3276
21:04:11.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:11.243425 23798 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
21:04:11.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:11.243541 23798 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
21:04:11.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:11.243649 23798 ts_tablet_manager.cc:616] Registered 0 tablets
21:04:11.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:11.243898 23798 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s user 0.001s sys 0.000s
21:04:11.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:11.305928 23798 rpc_server.cc:307] RPC server started. Bound to: 127.22.168.3:35191
21:04:11.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:11.305980 23935 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.22.168.3:35191 every 8 connection(s)
21:04:11.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:11.307364 23798 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskRxIrUU/test-tmp/mini-kudu-cluster8143392791847762722/ts-2/data/info.pb
21:04:11.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:11.310060 23937 heartbeater.cc:344] Connected to a master server at 127.22.168.62:37239
21:04:11.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:11.310521 23937 heartbeater.cc:461] Registering TS with master...
21:04:11.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:11.312217 23937 heartbeater.cc:507] Master 127.22.168.62:37239 requested a full tablet report, sending...
21:04:11.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:11.314086 23200 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskRxIrUU/build/asan/bin/kudu as pid 23798
21:04:11.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:11.314209 23242 ts_manager.cc:194] Registered new tserver with Master: e24858bd4ab14759b2252ddad7f0e5cd (127.22.168.3:35191)
21:04:11.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:11.314253 23200 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskRxIrUU/test-tmp/mini-kudu-cluster8143392791847762722/ts-2/wal/instance
21:04:11.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:11.322872 23936 heartbeater.cc:344] Connected to a master server at 127.22.168.60:33247
21:04:11.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:11.323335 23936 heartbeater.cc:461] Registering TS with master...
21:04:11.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:11.323906 23936 heartbeater.cc:507] Master 127.22.168.60:33247 requested a full tablet report, sending...
21:04:11.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:11.325219 23404 ts_manager.cc:194] Registered new tserver with Master: e24858bd4ab14759b2252ddad7f0e5cd (127.22.168.3:35191)
21:04:11.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:11.326242 23404 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.22.168.3:57375
21:04:11.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:11.326407 23938 heartbeater.cc:344] Connected to a master server at 127.22.168.61:32947
21:04:11.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:11.326545 23938 heartbeater.cc:461] Registering TS with master...
21:04:11.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:11.327078 23938 heartbeater.cc:507] Master 127.22.168.61:32947 requested a full tablet report, sending...
21:04:11.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:11.328156 23323 ts_manager.cc:194] Registered new tserver with Master: e24858bd4ab14759b2252ddad7f0e5cd (127.22.168.3:35191)
21:04:11.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:11.337330 23200 external_mini_cluster.cc:949] 3 TS(s) registered with all masters
21:04:11.339 [DEBUG - main] (MiniKuduCluster.java:195) Response:
21:04:11.339 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_masters {
}
21:04:11.340 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.22.168.62"
port: 37239
}
bound_http_address {
host: "127.22.168.62"
port: 33359
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.22.168.61"
port: 32947
}
bound_http_address {
host: "127.22.168.61"
port: 37179
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.22.168.60"
port: 33247
}
bound_http_address {
host: "127.22.168.60"
port: 33853
}
}
}
21:04:11.340 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_tservers {
}
21:04:11.340 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_tservers {
tservers {
id {
type: TSERVER
index: 0
}
bound_rpc_address {
host: "127.22.168.1"
port: 42241
}
bound_http_address {
host: "127.22.168.1"
port: 43979
}
}
tservers {
id {
type: TSERVER
index: 1
}
bound_rpc_address {
host: "127.22.168.2"
port: 37409
}
bound_http_address {
host: "127.22.168.2"
port: 39545
}
}
tservers {
id {
type: TSERVER
index: 2
}
bound_rpc_address {
host: "127.22.168.3"
port: 35191
}
bound_http_address {
host: "127.22.168.3"
port: 32991
}
}
}
21:04:11.341 [INFO - main] (KuduTestHarness.java:157) Creating a new Kudu client...
21:04:11.342 [DEBUG - main] (SecurityUtil.java:122) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication
21:04:11.344 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:11.345 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.22.168.60:33247(127.22.168.60:33247)] connecting to peer
21:04:11.346 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.22.168.62:37239(127.22.168.62:37239)] Successfully connected to peer
21:04:11.347 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:11.347 [DEBUG - kudu-nio-1] (Connection.java:757) [peer master-127.22.168.60:33247(127.22.168.60:33247)] Successfully connected to peer
21:04:11.349 [DEBUG - kudu-nio-1] (Negotiator.java:529) SASL mechanism PLAIN chosen for peer 127.22.168.60
21:04:11.352 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.22.168.61:32947(127.22.168.61:32947)] Successfully connected to peer
21:04:11.353 [DEBUG - kudu-nio-0] (Negotiator.java:529) SASL mechanism PLAIN chosen for peer 127.22.168.62
21:04:11.366 [DEBUG - kudu-nio-0] (Negotiator.java:529) SASL mechanism PLAIN chosen for peer 127.22.168.61
21:04:11.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:11.369589 23653 heartbeater.cc:499] Master 127.22.168.60:33247 was elected leader, sending a full tablet report...
21:04:11.372 [DEBUG - kudu-nio-1] (Negotiator.java:919) Authenticated connection [id: 0x73ff8a86, L:/127.0.0.1:48712 - R:/127.22.168.60:33247] using SASL/PLAIN
21:04:11.376 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet Kudu Master for table 'Kudu Master' with partition [<start>, <end>)
21:04:11.376 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=CreateTable, tablet=null, attempt=1, TimeoutTracker(timeout=50000, elapsed=33), Traces: [0ms] refreshing cache from master, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [2ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [4ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [31ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, deferred=null) after lookup
21:04:11.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:11.378515 23404 catalog_manager.cc:2257] Servicing CreateTable request from {username='slave'} at 127.0.0.1:48712:
21:04:11.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_manager_fallback_rolling"
21:04:11.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) schema {
21:04:11.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
21:04:11.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "key"
21:04:11.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT32
21:04:11.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
21:04:11.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
21:04:11.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
21:04:11.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
21:04:11.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_auto_incrementing: false
21:04:11.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
21:04:11.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
21:04:11.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "column1_i"
21:04:11.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT32
21:04:11.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
21:04:11.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
21:04:11.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
21:04:11.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
21:04:11.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_auto_incrementing: false
21:04:11.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
21:04:11.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
21:04:11.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "column2_i"
21:04:11.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT32
21:04:11.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
21:04:11.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
21:04:11.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
21:04:11.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
21:04:11.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_auto_incrementing: false
21:04:11.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
21:04:11.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
21:04:11.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "column3_s"
21:04:11.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
21:04:11.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
21:04:11.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: true
21:04:11.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: DICT_ENCODING
21:04:11.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: LZ4
21:04:11.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 4096
21:04:11.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
21:04:11.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_auto_incrementing: false
21:04:11.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
21:04:11.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
21:04:11.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "column4_b"
21:04:11.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: BOOL
21:04:11.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
21:04:11.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
21:04:11.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
21:04:11.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
21:04:11.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_auto_incrementing: false
21:04:11.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
21:04:11.380 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0xfbeb0198, L:/127.0.0.1:50562 - R:/127.22.168.62:37239] using SASL/PLAIN
21:04:11.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
21:04:11.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) partition_schema {
21:04:11.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) hash_schema {
21:04:11.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
21:04:11.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "key"
21:04:11.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
21:04:11.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) num_buckets: 2
21:04:11.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) seed: 0
21:04:11.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
21:04:11.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
21:04:11.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:11.380929 23404 catalog_manager.cc:7011] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table txn_manager_fallback_rolling in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
21:04:11.386 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0x3acf0785, L:/127.0.0.1:43940 - R:/127.22.168.61:32947] using SASL/PLAIN
21:04:11.397 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:894) Opened table 03fcdef52a6f4a7099b05a3d2fa91c76
21:04:11.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:11.434724 23587 tablet_service.cc:1505] Processing CreateTablet for tablet 260c684b258041bab84c3ff36578f96b (DEFAULT_TABLE table=txn_manager_fallback_rolling [id=03fcdef52a6f4a7099b05a3d2fa91c76]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
21:04:11.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:11.434772 23729 tablet_service.cc:1505] Processing CreateTablet for tablet 01aa0e192c434b468ec7ed90d48563b3 (DEFAULT_TABLE table=txn_manager_fallback_rolling [id=03fcdef52a6f4a7099b05a3d2fa91c76]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
21:04:11.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:11.434567 23870 tablet_service.cc:1505] Processing CreateTablet for tablet 260c684b258041bab84c3ff36578f96b (DEFAULT_TABLE table=txn_manager_fallback_rolling [id=03fcdef52a6f4a7099b05a3d2fa91c76]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
21:04:11.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:11.435452 23871 tablet_service.cc:1505] Processing CreateTablet for tablet 01aa0e192c434b468ec7ed90d48563b3 (DEFAULT_TABLE table=txn_manager_fallback_rolling [id=03fcdef52a6f4a7099b05a3d2fa91c76]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
21:04:11.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:11.436208 23729 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 01aa0e192c434b468ec7ed90d48563b3. 1 dirs total, 0 dirs full, 0 dirs failed
21:04:11.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:11.436208 23870 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 260c684b258041bab84c3ff36578f96b. 1 dirs total, 0 dirs full, 0 dirs failed
21:04:11.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:11.436092 23588 tablet_service.cc:1505] Processing CreateTablet for tablet 01aa0e192c434b468ec7ed90d48563b3 (DEFAULT_TABLE table=txn_manager_fallback_rolling [id=03fcdef52a6f4a7099b05a3d2fa91c76]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
21:04:11.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:11.436457 23587 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 260c684b258041bab84c3ff36578f96b. 1 dirs total, 0 dirs full, 0 dirs failed
21:04:11.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:11.434968 23728 tablet_service.cc:1505] Processing CreateTablet for tablet 260c684b258041bab84c3ff36578f96b (DEFAULT_TABLE table=txn_manager_fallback_rolling [id=03fcdef52a6f4a7099b05a3d2fa91c76]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
21:04:11.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:11.436890 23728 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 260c684b258041bab84c3ff36578f96b. 1 dirs total, 0 dirs full, 0 dirs failed
21:04:11.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:11.438386 23871 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 01aa0e192c434b468ec7ed90d48563b3. 1 dirs total, 0 dirs full, 0 dirs failed
21:04:11.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:11.439260 23588 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 01aa0e192c434b468ec7ed90d48563b3. 1 dirs total, 0 dirs full, 0 dirs failed
21:04:11.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:11.468962 23962 tablet_bootstrap.cc:492] T 260c684b258041bab84c3ff36578f96b P 1acec06b763242d082ddb75231fc4eca: Bootstrap starting.
21:04:11.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:11.469616 23963 tablet_bootstrap.cc:492] T 01aa0e192c434b468ec7ed90d48563b3 P 210cdae6f0e44d83acf5641ce563b738: Bootstrap starting.
21:04:11.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:11.473621 23963 tablet_bootstrap.cc:654] T 01aa0e192c434b468ec7ed90d48563b3 P 210cdae6f0e44d83acf5641ce563b738: Neither blocks nor log segments found. Creating new log.
21:04:11.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:11.473619 23962 tablet_bootstrap.cc:654] T 260c684b258041bab84c3ff36578f96b P 1acec06b763242d082ddb75231fc4eca: Neither blocks nor log segments found. Creating new log.
21:04:11.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:11.475042 23963 log.cc:826] T 01aa0e192c434b468ec7ed90d48563b3 P 210cdae6f0e44d83acf5641ce563b738: Log is configured to *not* fsync() on all Append() calls
21:04:11.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:11.475044 23962 log.cc:826] T 260c684b258041bab84c3ff36578f96b P 1acec06b763242d082ddb75231fc4eca: Log is configured to *not* fsync() on all Append() calls
21:04:11.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:11.476666 23964 tablet_bootstrap.cc:492] T 260c684b258041bab84c3ff36578f96b P e24858bd4ab14759b2252ddad7f0e5cd: Bootstrap starting.
21:04:11.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:11.480759 23964 tablet_bootstrap.cc:654] T 260c684b258041bab84c3ff36578f96b P e24858bd4ab14759b2252ddad7f0e5cd: Neither blocks nor log segments found. Creating new log.
21:04:11.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:11.482256 23964 log.cc:826] T 260c684b258041bab84c3ff36578f96b P e24858bd4ab14759b2252ddad7f0e5cd: Log is configured to *not* fsync() on all Append() calls
21:04:11.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:11.484229 23963 tablet_bootstrap.cc:492] T 01aa0e192c434b468ec7ed90d48563b3 P 210cdae6f0e44d83acf5641ce563b738: No bootstrap required, opened a new log
21:04:11.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:11.484666 23963 ts_tablet_manager.cc:1403] T 01aa0e192c434b468ec7ed90d48563b3 P 210cdae6f0e44d83acf5641ce563b738: Time spent bootstrapping tablet: real 0.015s user 0.010s sys 0.004s
21:04:11.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:11.485296 23962 tablet_bootstrap.cc:492] T 260c684b258041bab84c3ff36578f96b P 1acec06b763242d082ddb75231fc4eca: No bootstrap required, opened a new log
21:04:11.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:11.485668 23962 ts_tablet_manager.cc:1403] T 260c684b258041bab84c3ff36578f96b P 1acec06b763242d082ddb75231fc4eca: Time spent bootstrapping tablet: real 0.017s user 0.004s sys 0.011s
21:04:11.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:11.488005 23964 tablet_bootstrap.cc:492] T 260c684b258041bab84c3ff36578f96b P e24858bd4ab14759b2252ddad7f0e5cd: No bootstrap required, opened a new log
21:04:11.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:11.488310 23964 ts_tablet_manager.cc:1403] T 260c684b258041bab84c3ff36578f96b P e24858bd4ab14759b2252ddad7f0e5cd: Time spent bootstrapping tablet: real 0.012s user 0.004s sys 0.004s
21:04:11.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:11.492125 23963 raft_consensus.cc:359] T 01aa0e192c434b468ec7ed90d48563b3 P 210cdae6f0e44d83acf5641ce563b738 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "210cdae6f0e44d83acf5641ce563b738" member_type: VOTER last_known_addr { host: "127.22.168.1" port: 42241 } } peers { permanent_uuid: "e24858bd4ab14759b2252ddad7f0e5cd" member_type: VOTER last_known_addr { host: "127.22.168.3" port: 35191 } } peers { permanent_uuid: "1acec06b763242d082ddb75231fc4eca" member_type: VOTER last_known_addr { host: "127.22.168.2" port: 37409 } }
21:04:11.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:11.492579 23963 raft_consensus.cc:385] T 01aa0e192c434b468ec7ed90d48563b3 P 210cdae6f0e44d83acf5641ce563b738 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
21:04:11.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:11.492676 23963 raft_consensus.cc:740] T 01aa0e192c434b468ec7ed90d48563b3 P 210cdae6f0e44d83acf5641ce563b738 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 210cdae6f0e44d83acf5641ce563b738, State: Initialized, Role: FOLLOWER
21:04:11.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:11.492687 23962 raft_consensus.cc:359] T 260c684b258041bab84c3ff36578f96b P 1acec06b763242d082ddb75231fc4eca [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "210cdae6f0e44d83acf5641ce563b738" member_type: VOTER last_known_addr { host: "127.22.168.1" port: 42241 } } peers { permanent_uuid: "1acec06b763242d082ddb75231fc4eca" member_type: VOTER last_known_addr { host: "127.22.168.2" port: 37409 } } peers { permanent_uuid: "e24858bd4ab14759b2252ddad7f0e5cd" member_type: VOTER last_known_addr { host: "127.22.168.3" port: 35191 } }
21:04:11.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:11.493199 23962 raft_consensus.cc:385] T 260c684b258041bab84c3ff36578f96b P 1acec06b763242d082ddb75231fc4eca [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
21:04:11.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:11.493306 23962 raft_consensus.cc:740] T 260c684b258041bab84c3ff36578f96b P 1acec06b763242d082ddb75231fc4eca [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 1acec06b763242d082ddb75231fc4eca, State: Initialized, Role: FOLLOWER
21:04:11.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:11.493378 23963 consensus_queue.cc:260] T 01aa0e192c434b468ec7ed90d48563b3 P 210cdae6f0e44d83acf5641ce563b738 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "210cdae6f0e44d83acf5641ce563b738" member_type: VOTER last_known_addr { host: "127.22.168.1" port: 42241 } } peers { permanent_uuid: "e24858bd4ab14759b2252ddad7f0e5cd" member_type: VOTER last_known_addr { host: "127.22.168.3" port: 35191 } } peers { permanent_uuid: "1acec06b763242d082ddb75231fc4eca" member_type: VOTER last_known_addr { host: "127.22.168.2" port: 37409 } }
21:04:11.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:11.493700 23964 raft_consensus.cc:359] T 260c684b258041bab84c3ff36578f96b P e24858bd4ab14759b2252ddad7f0e5cd [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "210cdae6f0e44d83acf5641ce563b738" member_type: VOTER last_known_addr { host: "127.22.168.1" port: 42241 } } peers { permanent_uuid: "1acec06b763242d082ddb75231fc4eca" member_type: VOTER last_known_addr { host: "127.22.168.2" port: 37409 } } peers { permanent_uuid: "e24858bd4ab14759b2252ddad7f0e5cd" member_type: VOTER last_known_addr { host: "127.22.168.3" port: 35191 } }
21:04:11.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:11.493916 23962 consensus_queue.cc:260] T 260c684b258041bab84c3ff36578f96b P 1acec06b763242d082ddb75231fc4eca [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "210cdae6f0e44d83acf5641ce563b738" member_type: VOTER last_known_addr { host: "127.22.168.1" port: 42241 } } peers { permanent_uuid: "1acec06b763242d082ddb75231fc4eca" member_type: VOTER last_known_addr { host: "127.22.168.2" port: 37409 } } peers { permanent_uuid: "e24858bd4ab14759b2252ddad7f0e5cd" member_type: VOTER last_known_addr { host: "127.22.168.3" port: 35191 } }
21:04:11.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:11.494170 23964 raft_consensus.cc:385] T 260c684b258041bab84c3ff36578f96b P e24858bd4ab14759b2252ddad7f0e5cd [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
21:04:11.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:11.494275 23964 raft_consensus.cc:740] T 260c684b258041bab84c3ff36578f96b P e24858bd4ab14759b2252ddad7f0e5cd [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: e24858bd4ab14759b2252ddad7f0e5cd, State: Initialized, Role: FOLLOWER
21:04:11.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:11.494959 23964 consensus_queue.cc:260] T 260c684b258041bab84c3ff36578f96b P e24858bd4ab14759b2252ddad7f0e5cd [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "210cdae6f0e44d83acf5641ce563b738" member_type: VOTER last_known_addr { host: "127.22.168.1" port: 42241 } } peers { permanent_uuid: "1acec06b763242d082ddb75231fc4eca" member_type: VOTER last_known_addr { host: "127.22.168.2" port: 37409 } } peers { permanent_uuid: "e24858bd4ab14759b2252ddad7f0e5cd" member_type: VOTER last_known_addr { host: "127.22.168.3" port: 35191 } }
21:04:11.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:11.495385 23963 ts_tablet_manager.cc:1434] T 01aa0e192c434b468ec7ed90d48563b3 P 210cdae6f0e44d83acf5641ce563b738: Time spent starting tablet: real 0.010s user 0.006s sys 0.004s
21:04:11.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:11.496840 23963 tablet_bootstrap.cc:492] T 260c684b258041bab84c3ff36578f96b P 210cdae6f0e44d83acf5641ce563b738: Bootstrap starting.
21:04:11.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:11.498749 23963 tablet_bootstrap.cc:654] T 260c684b258041bab84c3ff36578f96b P 210cdae6f0e44d83acf5641ce563b738: Neither blocks nor log segments found. Creating new log.
21:04:11.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:11.502746 23794 heartbeater.cc:499] Master 127.22.168.60:33247 was elected leader, sending a full tablet report...
21:04:11.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:11.503394 23963 tablet_bootstrap.cc:492] T 260c684b258041bab84c3ff36578f96b P 210cdae6f0e44d83acf5641ce563b738: No bootstrap required, opened a new log
21:04:11.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:11.503541 23963 ts_tablet_manager.cc:1403] T 260c684b258041bab84c3ff36578f96b P 210cdae6f0e44d83acf5641ce563b738: Time spent bootstrapping tablet: real 0.007s user 0.001s sys 0.003s
21:04:11.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:11.504508 23963 raft_consensus.cc:359] T 260c684b258041bab84c3ff36578f96b P 210cdae6f0e44d83acf5641ce563b738 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "210cdae6f0e44d83acf5641ce563b738" member_type: VOTER last_known_addr { host: "127.22.168.1" port: 42241 } } peers { permanent_uuid: "1acec06b763242d082ddb75231fc4eca" member_type: VOTER last_known_addr { host: "127.22.168.2" port: 37409 } } peers { permanent_uuid: "e24858bd4ab14759b2252ddad7f0e5cd" member_type: VOTER last_known_addr { host: "127.22.168.3" port: 35191 } }
21:04:11.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:11.504695 23963 raft_consensus.cc:385] T 260c684b258041bab84c3ff36578f96b P 210cdae6f0e44d83acf5641ce563b738 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
21:04:11.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:11.504789 23963 raft_consensus.cc:740] T 260c684b258041bab84c3ff36578f96b P 210cdae6f0e44d83acf5641ce563b738 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 210cdae6f0e44d83acf5641ce563b738, State: Initialized, Role: FOLLOWER
21:04:11.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:11.505069 23963 consensus_queue.cc:260] T 260c684b258041bab84c3ff36578f96b P 210cdae6f0e44d83acf5641ce563b738 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "210cdae6f0e44d83acf5641ce563b738" member_type: VOTER last_known_addr { host: "127.22.168.1" port: 42241 } } peers { permanent_uuid: "1acec06b763242d082ddb75231fc4eca" member_type: VOTER last_known_addr { host: "127.22.168.2" port: 37409 } } peers { permanent_uuid: "e24858bd4ab14759b2252ddad7f0e5cd" member_type: VOTER last_known_addr { host: "127.22.168.3" port: 35191 } }
21:04:11.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:11.505708 23963 ts_tablet_manager.cc:1434] T 260c684b258041bab84c3ff36578f96b P 210cdae6f0e44d83acf5641ce563b738: Time spent starting tablet: real 0.002s user 0.002s sys 0.000s
21:04:11.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:11.506695 23936 heartbeater.cc:499] Master 127.22.168.60:33247 was elected leader, sending a full tablet report...
21:04:11.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:11.507575 23964 ts_tablet_manager.cc:1434] T 260c684b258041bab84c3ff36578f96b P e24858bd4ab14759b2252ddad7f0e5cd: Time spent starting tablet: real 0.019s user 0.007s sys 0.007s
21:04:11.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:11.508922 23964 tablet_bootstrap.cc:492] T 01aa0e192c434b468ec7ed90d48563b3 P e24858bd4ab14759b2252ddad7f0e5cd: Bootstrap starting.
21:04:11.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:11.511004 23964 tablet_bootstrap.cc:654] T 01aa0e192c434b468ec7ed90d48563b3 P e24858bd4ab14759b2252ddad7f0e5cd: Neither blocks nor log segments found. Creating new log.
21:04:11.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:11.513617 23962 ts_tablet_manager.cc:1434] T 260c684b258041bab84c3ff36578f96b P 1acec06b763242d082ddb75231fc4eca: Time spent starting tablet: real 0.028s user 0.004s sys 0.009s
21:04:11.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:11.514617 23964 tablet_bootstrap.cc:492] T 01aa0e192c434b468ec7ed90d48563b3 P e24858bd4ab14759b2252ddad7f0e5cd: No bootstrap required, opened a new log
21:04:11.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:11.514722 23962 tablet_bootstrap.cc:492] T 01aa0e192c434b468ec7ed90d48563b3 P 1acec06b763242d082ddb75231fc4eca: Bootstrap starting.
21:04:11.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:11.514926 23964 ts_tablet_manager.cc:1403] T 01aa0e192c434b468ec7ed90d48563b3 P e24858bd4ab14759b2252ddad7f0e5cd: Time spent bootstrapping tablet: real 0.006s user 0.001s sys 0.004s
21:04:11.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:11.515925 23964 raft_consensus.cc:359] T 01aa0e192c434b468ec7ed90d48563b3 P e24858bd4ab14759b2252ddad7f0e5cd [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "210cdae6f0e44d83acf5641ce563b738" member_type: VOTER last_known_addr { host: "127.22.168.1" port: 42241 } } peers { permanent_uuid: "e24858bd4ab14759b2252ddad7f0e5cd" member_type: VOTER last_known_addr { host: "127.22.168.3" port: 35191 } } peers { permanent_uuid: "1acec06b763242d082ddb75231fc4eca" member_type: VOTER last_known_addr { host: "127.22.168.2" port: 37409 } }
21:04:11.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:11.516646 23962 tablet_bootstrap.cc:654] T 01aa0e192c434b468ec7ed90d48563b3 P 1acec06b763242d082ddb75231fc4eca: Neither blocks nor log segments found. Creating new log.
21:04:11.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:11.516685 23964 raft_consensus.cc:385] T 01aa0e192c434b468ec7ed90d48563b3 P e24858bd4ab14759b2252ddad7f0e5cd [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
21:04:11.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:11.516779 23964 raft_consensus.cc:740] T 01aa0e192c434b468ec7ed90d48563b3 P e24858bd4ab14759b2252ddad7f0e5cd [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: e24858bd4ab14759b2252ddad7f0e5cd, State: Initialized, Role: FOLLOWER
21:04:11.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:11.517076 23964 consensus_queue.cc:260] T 01aa0e192c434b468ec7ed90d48563b3 P e24858bd4ab14759b2252ddad7f0e5cd [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "210cdae6f0e44d83acf5641ce563b738" member_type: VOTER last_known_addr { host: "127.22.168.1" port: 42241 } } peers { permanent_uuid: "e24858bd4ab14759b2252ddad7f0e5cd" member_type: VOTER last_known_addr { host: "127.22.168.3" port: 35191 } } peers { permanent_uuid: "1acec06b763242d082ddb75231fc4eca" member_type: VOTER last_known_addr { host: "127.22.168.2" port: 37409 } }
21:04:11.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:11.517829 23964 ts_tablet_manager.cc:1434] T 01aa0e192c434b468ec7ed90d48563b3 P e24858bd4ab14759b2252ddad7f0e5cd: Time spent starting tablet: real 0.003s user 0.000s sys 0.002s
21:04:11.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:11.521004 23962 tablet_bootstrap.cc:492] T 01aa0e192c434b468ec7ed90d48563b3 P 1acec06b763242d082ddb75231fc4eca: No bootstrap required, opened a new log
21:04:11.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:11.521163 23962 ts_tablet_manager.cc:1403] T 01aa0e192c434b468ec7ed90d48563b3 P 1acec06b763242d082ddb75231fc4eca: Time spent bootstrapping tablet: real 0.007s user 0.000s sys 0.005s
21:04:11.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:11.521901 23962 raft_consensus.cc:359] T 01aa0e192c434b468ec7ed90d48563b3 P 1acec06b763242d082ddb75231fc4eca [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "210cdae6f0e44d83acf5641ce563b738" member_type: VOTER last_known_addr { host: "127.22.168.1" port: 42241 } } peers { permanent_uuid: "e24858bd4ab14759b2252ddad7f0e5cd" member_type: VOTER last_known_addr { host: "127.22.168.3" port: 35191 } } peers { permanent_uuid: "1acec06b763242d082ddb75231fc4eca" member_type: VOTER last_known_addr { host: "127.22.168.2" port: 37409 } }
21:04:11.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:11.522045 23962 raft_consensus.cc:385] T 01aa0e192c434b468ec7ed90d48563b3 P 1acec06b763242d082ddb75231fc4eca [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
21:04:11.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:11.522130 23962 raft_consensus.cc:740] T 01aa0e192c434b468ec7ed90d48563b3 P 1acec06b763242d082ddb75231fc4eca [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 1acec06b763242d082ddb75231fc4eca, State: Initialized, Role: FOLLOWER
21:04:11.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:11.522367 23962 consensus_queue.cc:260] T 01aa0e192c434b468ec7ed90d48563b3 P 1acec06b763242d082ddb75231fc4eca [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "210cdae6f0e44d83acf5641ce563b738" member_type: VOTER last_known_addr { host: "127.22.168.1" port: 42241 } } peers { permanent_uuid: "e24858bd4ab14759b2252ddad7f0e5cd" member_type: VOTER last_known_addr { host: "127.22.168.3" port: 35191 } } peers { permanent_uuid: "1acec06b763242d082ddb75231fc4eca" member_type: VOTER last_known_addr { host: "127.22.168.2" port: 37409 } }
21:04:11.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:11.522841 23962 ts_tablet_manager.cc:1434] T 01aa0e192c434b468ec7ed90d48563b3 P 1acec06b763242d082ddb75231fc4eca: Time spent starting tablet: real 0.002s user 0.000s sys 0.000s
21:04:11.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:11.574077 23939 tablet.cc:2378] T 01aa0e192c434b468ec7ed90d48563b3 P e24858bd4ab14759b2252ddad7f0e5cd: Can't schedule compaction. Clean time has not been advanced past its initial value.
21:04:11.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:11.589799 23799 tablet.cc:2378] T 01aa0e192c434b468ec7ed90d48563b3 P 1acec06b763242d082ddb75231fc4eca: Can't schedule compaction. Clean time has not been advanced past its initial value.
21:04:11.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:11.606149 23656 tablet.cc:2378] T 01aa0e192c434b468ec7ed90d48563b3 P 210cdae6f0e44d83acf5641ce563b738: Can't schedule compaction. Clean time has not been advanced past its initial value.
21:04:11.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:11.609323 23969 raft_consensus.cc:493] T 260c684b258041bab84c3ff36578f96b P 1acec06b763242d082ddb75231fc4eca [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
21:04:11.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:11.609555 23969 raft_consensus.cc:515] T 260c684b258041bab84c3ff36578f96b P 1acec06b763242d082ddb75231fc4eca [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "210cdae6f0e44d83acf5641ce563b738" member_type: VOTER last_known_addr { host: "127.22.168.1" port: 42241 } } peers { permanent_uuid: "1acec06b763242d082ddb75231fc4eca" member_type: VOTER last_known_addr { host: "127.22.168.2" port: 37409 } } peers { permanent_uuid: "e24858bd4ab14759b2252ddad7f0e5cd" member_type: VOTER last_known_addr { host: "127.22.168.3" port: 35191 } }
21:04:11.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:11.611225 23969 leader_election.cc:290] T 260c684b258041bab84c3ff36578f96b P 1acec06b763242d082ddb75231fc4eca [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 210cdae6f0e44d83acf5641ce563b738 (127.22.168.1:42241), e24858bd4ab14759b2252ddad7f0e5cd (127.22.168.3:35191)
21:04:11.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:11.618847 23891 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "260c684b258041bab84c3ff36578f96b" candidate_uuid: "1acec06b763242d082ddb75231fc4eca" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "e24858bd4ab14759b2252ddad7f0e5cd" is_pre_election: true
21:04:11.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:11.618873 23608 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "260c684b258041bab84c3ff36578f96b" candidate_uuid: "1acec06b763242d082ddb75231fc4eca" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "210cdae6f0e44d83acf5641ce563b738" is_pre_election: true
21:04:11.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:11.619354 23608 raft_consensus.cc:2468] T 260c684b258041bab84c3ff36578f96b P 210cdae6f0e44d83acf5641ce563b738 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 1acec06b763242d082ddb75231fc4eca in term 0.
21:04:11.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:11.619393 23891 raft_consensus.cc:2468] T 260c684b258041bab84c3ff36578f96b P e24858bd4ab14759b2252ddad7f0e5cd [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 1acec06b763242d082ddb75231fc4eca in term 0.
21:04:11.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:11.620010 23673 leader_election.cc:304] T 260c684b258041bab84c3ff36578f96b P 1acec06b763242d082ddb75231fc4eca [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 1acec06b763242d082ddb75231fc4eca, e24858bd4ab14759b2252ddad7f0e5cd; no voters:
21:04:11.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:11.620500 23969 raft_consensus.cc:2804] T 260c684b258041bab84c3ff36578f96b P 1acec06b763242d082ddb75231fc4eca [term 0 FOLLOWER]: Leader pre-election won for term 1
21:04:11.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:11.620663 23969 raft_consensus.cc:493] T 260c684b258041bab84c3ff36578f96b P 1acec06b763242d082ddb75231fc4eca [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
21:04:11.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:11.620771 23969 raft_consensus.cc:3060] T 260c684b258041bab84c3ff36578f96b P 1acec06b763242d082ddb75231fc4eca [term 0 FOLLOWER]: Advancing to term 1
21:04:11.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:11.622888 23969 raft_consensus.cc:515] T 260c684b258041bab84c3ff36578f96b P 1acec06b763242d082ddb75231fc4eca [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "210cdae6f0e44d83acf5641ce563b738" member_type: VOTER last_known_addr { host: "127.22.168.1" port: 42241 } } peers { permanent_uuid: "1acec06b763242d082ddb75231fc4eca" member_type: VOTER last_known_addr { host: "127.22.168.2" port: 37409 } } peers { permanent_uuid: "e24858bd4ab14759b2252ddad7f0e5cd" member_type: VOTER last_known_addr { host: "127.22.168.3" port: 35191 } }
21:04:11.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:11.623535 23969 leader_election.cc:290] T 260c684b258041bab84c3ff36578f96b P 1acec06b763242d082ddb75231fc4eca [CANDIDATE]: Term 1 election: Requested vote from peers 210cdae6f0e44d83acf5641ce563b738 (127.22.168.1:42241), e24858bd4ab14759b2252ddad7f0e5cd (127.22.168.3:35191)
21:04:11.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:11.624061 23608 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "260c684b258041bab84c3ff36578f96b" candidate_uuid: "1acec06b763242d082ddb75231fc4eca" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "210cdae6f0e44d83acf5641ce563b738"
21:04:11.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:11.624107 23891 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "260c684b258041bab84c3ff36578f96b" candidate_uuid: "1acec06b763242d082ddb75231fc4eca" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "e24858bd4ab14759b2252ddad7f0e5cd"
21:04:11.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:11.624270 23891 raft_consensus.cc:3060] T 260c684b258041bab84c3ff36578f96b P e24858bd4ab14759b2252ddad7f0e5cd [term 0 FOLLOWER]: Advancing to term 1
21:04:11.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:11.624259 23608 raft_consensus.cc:3060] T 260c684b258041bab84c3ff36578f96b P 210cdae6f0e44d83acf5641ce563b738 [term 0 FOLLOWER]: Advancing to term 1
21:04:11.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:11.626515 23891 raft_consensus.cc:2468] T 260c684b258041bab84c3ff36578f96b P e24858bd4ab14759b2252ddad7f0e5cd [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 1acec06b763242d082ddb75231fc4eca in term 1.
21:04:11.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:11.626518 23608 raft_consensus.cc:2468] T 260c684b258041bab84c3ff36578f96b P 210cdae6f0e44d83acf5641ce563b738 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 1acec06b763242d082ddb75231fc4eca in term 1.
21:04:11.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:11.626988 23673 leader_election.cc:304] T 260c684b258041bab84c3ff36578f96b P 1acec06b763242d082ddb75231fc4eca [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 3 yes votes; 0 no votes. yes voters: 1acec06b763242d082ddb75231fc4eca, 210cdae6f0e44d83acf5641ce563b738, e24858bd4ab14759b2252ddad7f0e5cd; no voters:
21:04:11.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:11.627306 23969 raft_consensus.cc:2804] T 260c684b258041bab84c3ff36578f96b P 1acec06b763242d082ddb75231fc4eca [term 1 FOLLOWER]: Leader election won for term 1
21:04:11.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:11.627784 23969 raft_consensus.cc:697] T 260c684b258041bab84c3ff36578f96b P 1acec06b763242d082ddb75231fc4eca [term 1 LEADER]: Becoming Leader. State: Replica: 1acec06b763242d082ddb75231fc4eca, State: Running, Role: LEADER
21:04:11.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:11.628312 23969 consensus_queue.cc:237] T 260c684b258041bab84c3ff36578f96b P 1acec06b763242d082ddb75231fc4eca [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "210cdae6f0e44d83acf5641ce563b738" member_type: VOTER last_known_addr { host: "127.22.168.1" port: 42241 } } peers { permanent_uuid: "1acec06b763242d082ddb75231fc4eca" member_type: VOTER last_known_addr { host: "127.22.168.2" port: 37409 } } peers { permanent_uuid: "e24858bd4ab14759b2252ddad7f0e5cd" member_type: VOTER last_known_addr { host: "127.22.168.3" port: 35191 } }
21:04:11.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:11.633672 23404 catalog_manager.cc:5649] T 260c684b258041bab84c3ff36578f96b P 1acec06b763242d082ddb75231fc4eca reported cstate change: term changed from 0 to 1, leader changed from <none> to 1acec06b763242d082ddb75231fc4eca (127.22.168.2). New cstate: current_term: 1 leader_uuid: "1acec06b763242d082ddb75231fc4eca" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "210cdae6f0e44d83acf5641ce563b738" member_type: VOTER last_known_addr { host: "127.22.168.1" port: 42241 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "1acec06b763242d082ddb75231fc4eca" member_type: VOTER last_known_addr { host: "127.22.168.2" port: 37409 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "e24858bd4ab14759b2252ddad7f0e5cd" member_type: VOTER last_known_addr { host: "127.22.168.3" port: 35191 } health_report { overall_health: UNKNOWN } } }
21:04:11.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:11.715746 23970 raft_consensus.cc:493] T 01aa0e192c434b468ec7ed90d48563b3 P e24858bd4ab14759b2252ddad7f0e5cd [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
21:04:11.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:11.716006 23970 raft_consensus.cc:515] T 01aa0e192c434b468ec7ed90d48563b3 P e24858bd4ab14759b2252ddad7f0e5cd [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "210cdae6f0e44d83acf5641ce563b738" member_type: VOTER last_known_addr { host: "127.22.168.1" port: 42241 } } peers { permanent_uuid: "e24858bd4ab14759b2252ddad7f0e5cd" member_type: VOTER last_known_addr { host: "127.22.168.3" port: 35191 } } peers { permanent_uuid: "1acec06b763242d082ddb75231fc4eca" member_type: VOTER last_known_addr { host: "127.22.168.2" port: 37409 } }
21:04:11.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:11.718147 23970 leader_election.cc:290] T 01aa0e192c434b468ec7ed90d48563b3 P e24858bd4ab14759b2252ddad7f0e5cd [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 210cdae6f0e44d83acf5641ce563b738 (127.22.168.1:42241), 1acec06b763242d082ddb75231fc4eca (127.22.168.2:37409)
21:04:11.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:11.726128 23608 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "01aa0e192c434b468ec7ed90d48563b3" candidate_uuid: "e24858bd4ab14759b2252ddad7f0e5cd" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "210cdae6f0e44d83acf5641ce563b738" is_pre_election: true
21:04:11.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:11.726401 23608 raft_consensus.cc:2468] T 01aa0e192c434b468ec7ed90d48563b3 P 210cdae6f0e44d83acf5641ce563b738 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate e24858bd4ab14759b2252ddad7f0e5cd in term 0.
21:04:11.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:11.726286 23749 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "01aa0e192c434b468ec7ed90d48563b3" candidate_uuid: "e24858bd4ab14759b2252ddad7f0e5cd" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "1acec06b763242d082ddb75231fc4eca" is_pre_election: true
21:04:11.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:11.726728 23749 raft_consensus.cc:2468] T 01aa0e192c434b468ec7ed90d48563b3 P 1acec06b763242d082ddb75231fc4eca [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate e24858bd4ab14759b2252ddad7f0e5cd in term 0.
21:04:11.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:11.726985 23817 leader_election.cc:304] T 01aa0e192c434b468ec7ed90d48563b3 P e24858bd4ab14759b2252ddad7f0e5cd [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 210cdae6f0e44d83acf5641ce563b738, e24858bd4ab14759b2252ddad7f0e5cd; no voters:
21:04:11.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:11.727434 23970 raft_consensus.cc:2804] T 01aa0e192c434b468ec7ed90d48563b3 P e24858bd4ab14759b2252ddad7f0e5cd [term 0 FOLLOWER]: Leader pre-election won for term 1
21:04:11.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:11.727540 23970 raft_consensus.cc:493] T 01aa0e192c434b468ec7ed90d48563b3 P e24858bd4ab14759b2252ddad7f0e5cd [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
21:04:11.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:11.727608 23970 raft_consensus.cc:3060] T 01aa0e192c434b468ec7ed90d48563b3 P e24858bd4ab14759b2252ddad7f0e5cd [term 0 FOLLOWER]: Advancing to term 1
21:04:11.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:11.729655 23970 raft_consensus.cc:515] T 01aa0e192c434b468ec7ed90d48563b3 P e24858bd4ab14759b2252ddad7f0e5cd [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "210cdae6f0e44d83acf5641ce563b738" member_type: VOTER last_known_addr { host: "127.22.168.1" port: 42241 } } peers { permanent_uuid: "e24858bd4ab14759b2252ddad7f0e5cd" member_type: VOTER last_known_addr { host: "127.22.168.3" port: 35191 } } peers { permanent_uuid: "1acec06b763242d082ddb75231fc4eca" member_type: VOTER last_known_addr { host: "127.22.168.2" port: 37409 } }
21:04:11.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:11.730201 23970 leader_election.cc:290] T 01aa0e192c434b468ec7ed90d48563b3 P e24858bd4ab14759b2252ddad7f0e5cd [CANDIDATE]: Term 1 election: Requested vote from peers 210cdae6f0e44d83acf5641ce563b738 (127.22.168.1:42241), 1acec06b763242d082ddb75231fc4eca (127.22.168.2:37409)
21:04:11.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:11.730722 23608 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "01aa0e192c434b468ec7ed90d48563b3" candidate_uuid: "e24858bd4ab14759b2252ddad7f0e5cd" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "210cdae6f0e44d83acf5641ce563b738"
21:04:11.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:11.730827 23749 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "01aa0e192c434b468ec7ed90d48563b3" candidate_uuid: "e24858bd4ab14759b2252ddad7f0e5cd" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "1acec06b763242d082ddb75231fc4eca"
21:04:11.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:11.730970 23749 raft_consensus.cc:3060] T 01aa0e192c434b468ec7ed90d48563b3 P 1acec06b763242d082ddb75231fc4eca [term 0 FOLLOWER]: Advancing to term 1
21:04:11.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:11.730954 23608 raft_consensus.cc:3060] T 01aa0e192c434b468ec7ed90d48563b3 P 210cdae6f0e44d83acf5641ce563b738 [term 0 FOLLOWER]: Advancing to term 1
21:04:11.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:11.732923 23608 raft_consensus.cc:2468] T 01aa0e192c434b468ec7ed90d48563b3 P 210cdae6f0e44d83acf5641ce563b738 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate e24858bd4ab14759b2252ddad7f0e5cd in term 1.
21:04:11.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:11.732942 23749 raft_consensus.cc:2468] T 01aa0e192c434b468ec7ed90d48563b3 P 1acec06b763242d082ddb75231fc4eca [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate e24858bd4ab14759b2252ddad7f0e5cd in term 1.
21:04:11.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:11.733278 23817 leader_election.cc:304] T 01aa0e192c434b468ec7ed90d48563b3 P e24858bd4ab14759b2252ddad7f0e5cd [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 210cdae6f0e44d83acf5641ce563b738, e24858bd4ab14759b2252ddad7f0e5cd; no voters:
21:04:11.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:11.733563 23970 raft_consensus.cc:2804] T 01aa0e192c434b468ec7ed90d48563b3 P e24858bd4ab14759b2252ddad7f0e5cd [term 1 FOLLOWER]: Leader election won for term 1
21:04:11.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:11.733927 23970 raft_consensus.cc:697] T 01aa0e192c434b468ec7ed90d48563b3 P e24858bd4ab14759b2252ddad7f0e5cd [term 1 LEADER]: Becoming Leader. State: Replica: e24858bd4ab14759b2252ddad7f0e5cd, State: Running, Role: LEADER
21:04:11.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:11.734460 23970 consensus_queue.cc:237] T 01aa0e192c434b468ec7ed90d48563b3 P e24858bd4ab14759b2252ddad7f0e5cd [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "210cdae6f0e44d83acf5641ce563b738" member_type: VOTER last_known_addr { host: "127.22.168.1" port: 42241 } } peers { permanent_uuid: "e24858bd4ab14759b2252ddad7f0e5cd" member_type: VOTER last_known_addr { host: "127.22.168.3" port: 35191 } } peers { permanent_uuid: "1acec06b763242d082ddb75231fc4eca" member_type: VOTER last_known_addr { host: "127.22.168.2" port: 37409 } }
21:04:11.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:11.739589 23404 catalog_manager.cc:5649] T 01aa0e192c434b468ec7ed90d48563b3 P e24858bd4ab14759b2252ddad7f0e5cd reported cstate change: term changed from 0 to 1, leader changed from <none> to e24858bd4ab14759b2252ddad7f0e5cd (127.22.168.3). New cstate: current_term: 1 leader_uuid: "e24858bd4ab14759b2252ddad7f0e5cd" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "210cdae6f0e44d83acf5641ce563b738" member_type: VOTER last_known_addr { host: "127.22.168.1" port: 42241 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "e24858bd4ab14759b2252ddad7f0e5cd" member_type: VOTER last_known_addr { host: "127.22.168.3" port: 35191 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "1acec06b763242d082ddb75231fc4eca" member_type: VOTER last_known_addr { host: "127.22.168.2" port: 37409 } health_report { overall_health: UNKNOWN } } }
21:04:11.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:11.894214 23404 catalog_manager.cc:2257] Servicing CreateTable request from {username='slave'} at 127.0.0.1:48722:
21:04:11.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "kudu_system.kudu_transactions"
21:04:11.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) schema {
21:04:11.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
21:04:11.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_id"
21:04:11.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT64
21:04:11.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
21:04:11.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
21:04:11.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
21:04:11.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
21:04:11.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
21:04:11.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
21:04:11.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
21:04:11.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
21:04:11.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "entry_type"
21:04:11.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT8
21:04:11.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
21:04:11.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
21:04:11.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
21:04:11.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
21:04:11.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
21:04:11.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
21:04:11.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
21:04:11.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
21:04:11.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "identifier"
21:04:11.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
21:04:11.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
21:04:11.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
21:04:11.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
21:04:11.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
21:04:11.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
21:04:11.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
21:04:11.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
21:04:11.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
21:04:11.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "metadata"
21:04:11.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
21:04:11.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
21:04:11.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
21:04:11.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
21:04:11.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
21:04:11.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
21:04:11.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
21:04:11.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
21:04:11.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
21:04:11.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) num_replicas: 3
21:04:11.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) split_rows_range_bounds {
21:04:11.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) rows: "<redacted>""\006\001\000\000\000\000\000\000\000\000\007\001@B\017\000\000\000\000\000"
21:04:11.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) indirect_data: "<redacted>"""
21:04:11.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
21:04:11.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) partition_schema {
21:04:11.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) range_schema {
21:04:11.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
21:04:11.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_id"
21:04:11.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
21:04:11.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
21:04:11.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
21:04:11.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) table_type: TXN_STATUS_TABLE
21:04:11.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:11.895480 23404 catalog_manager.cc:7011] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table kudu_system.kudu_transactions in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
21:04:11.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:11.907769 23870 tablet_service.cc:1505] Processing CreateTablet for tablet 581dbe2f34ca4a7981755fa5f4b6b6ca (TXN_STATUS_TABLE table=kudu_system.kudu_transactions [id=c54eb72594a5488d8dd9c9882da41ff4]), partition=RANGE (txn_id) PARTITION 0 <= VALUES < 1000000
21:04:11.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:11.907816 23588 tablet_service.cc:1505] Processing CreateTablet for tablet 581dbe2f34ca4a7981755fa5f4b6b6ca (TXN_STATUS_TABLE table=kudu_system.kudu_transactions [id=c54eb72594a5488d8dd9c9882da41ff4]), partition=RANGE (txn_id) PARTITION 0 <= VALUES < 1000000
21:04:11.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:11.908023 23728 tablet_service.cc:1505] Processing CreateTablet for tablet 581dbe2f34ca4a7981755fa5f4b6b6ca (TXN_STATUS_TABLE table=kudu_system.kudu_transactions [id=c54eb72594a5488d8dd9c9882da41ff4]), partition=RANGE (txn_id) PARTITION 0 <= VALUES < 1000000
21:04:11.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:11.908867 23588 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 581dbe2f34ca4a7981755fa5f4b6b6ca. 1 dirs total, 0 dirs full, 0 dirs failed
21:04:11.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:11.908867 23870 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 581dbe2f34ca4a7981755fa5f4b6b6ca. 1 dirs total, 0 dirs full, 0 dirs failed
21:04:11.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:11.909022 23728 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 581dbe2f34ca4a7981755fa5f4b6b6ca. 1 dirs total, 0 dirs full, 0 dirs failed
21:04:11.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:11.914794 23962 tablet_bootstrap.cc:492] T 581dbe2f34ca4a7981755fa5f4b6b6ca P 1acec06b763242d082ddb75231fc4eca: Bootstrap starting.
21:04:11.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:11.916582 23962 tablet_bootstrap.cc:654] T 581dbe2f34ca4a7981755fa5f4b6b6ca P 1acec06b763242d082ddb75231fc4eca: Neither blocks nor log segments found. Creating new log.
21:04:11.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:11.916711 23963 tablet_bootstrap.cc:492] T 581dbe2f34ca4a7981755fa5f4b6b6ca P 210cdae6f0e44d83acf5641ce563b738: Bootstrap starting.
21:04:11.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:11.917555 23964 tablet_bootstrap.cc:492] T 581dbe2f34ca4a7981755fa5f4b6b6ca P e24858bd4ab14759b2252ddad7f0e5cd: Bootstrap starting.
21:04:11.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:11.918386 23963 tablet_bootstrap.cc:654] T 581dbe2f34ca4a7981755fa5f4b6b6ca P 210cdae6f0e44d83acf5641ce563b738: Neither blocks nor log segments found. Creating new log.
21:04:11.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:11.918753 23962 tablet_bootstrap.cc:492] T 581dbe2f34ca4a7981755fa5f4b6b6ca P 1acec06b763242d082ddb75231fc4eca: No bootstrap required, opened a new log
21:04:11.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:11.918929 23962 ts_tablet_manager.cc:1403] T 581dbe2f34ca4a7981755fa5f4b6b6ca P 1acec06b763242d082ddb75231fc4eca: Time spent bootstrapping tablet: real 0.004s user 0.001s sys 0.002s
21:04:11.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:11.919193 23964 tablet_bootstrap.cc:654] T 581dbe2f34ca4a7981755fa5f4b6b6ca P e24858bd4ab14759b2252ddad7f0e5cd: Neither blocks nor log segments found. Creating new log.
21:04:11.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:11.919749 23962 raft_consensus.cc:359] T 581dbe2f34ca4a7981755fa5f4b6b6ca P 1acec06b763242d082ddb75231fc4eca [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "210cdae6f0e44d83acf5641ce563b738" member_type: VOTER last_known_addr { host: "127.22.168.1" port: 42241 } } peers { permanent_uuid: "1acec06b763242d082ddb75231fc4eca" member_type: VOTER last_known_addr { host: "127.22.168.2" port: 37409 } } peers { permanent_uuid: "e24858bd4ab14759b2252ddad7f0e5cd" member_type: VOTER last_known_addr { host: "127.22.168.3" port: 35191 } }
21:04:11.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:11.919966 23962 raft_consensus.cc:385] T 581dbe2f34ca4a7981755fa5f4b6b6ca P 1acec06b763242d082ddb75231fc4eca [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
21:04:11.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:11.920044 23962 raft_consensus.cc:740] T 581dbe2f34ca4a7981755fa5f4b6b6ca P 1acec06b763242d082ddb75231fc4eca [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 1acec06b763242d082ddb75231fc4eca, State: Initialized, Role: FOLLOWER
21:04:11.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:11.920343 23963 tablet_bootstrap.cc:492] T 581dbe2f34ca4a7981755fa5f4b6b6ca P 210cdae6f0e44d83acf5641ce563b738: No bootstrap required, opened a new log
21:04:11.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:11.920334 23962 consensus_queue.cc:260] T 581dbe2f34ca4a7981755fa5f4b6b6ca P 1acec06b763242d082ddb75231fc4eca [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "210cdae6f0e44d83acf5641ce563b738" member_type: VOTER last_known_addr { host: "127.22.168.1" port: 42241 } } peers { permanent_uuid: "1acec06b763242d082ddb75231fc4eca" member_type: VOTER last_known_addr { host: "127.22.168.2" port: 37409 } } peers { permanent_uuid: "e24858bd4ab14759b2252ddad7f0e5cd" member_type: VOTER last_known_addr { host: "127.22.168.3" port: 35191 } }
21:04:11.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:11.920536 23963 ts_tablet_manager.cc:1403] T 581dbe2f34ca4a7981755fa5f4b6b6ca P 210cdae6f0e44d83acf5641ce563b738: Time spent bootstrapping tablet: real 0.004s user 0.002s sys 0.001s
21:04:11.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:11.920814 23969 tablet_replica.cc:442] T 581dbe2f34ca4a7981755fa5f4b6b6ca P 1acec06b763242d082ddb75231fc4eca: TxnStatusTablet state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "210cdae6f0e44d83acf5641ce563b738" member_type: VOTER last_known_addr { host: "127.22.168.1" port: 42241 } } peers { permanent_uuid: "1acec06b763242d082ddb75231fc4eca" member_type: VOTER last_known_addr { host: "127.22.168.2" port: 37409 } } peers { permanent_uuid: "e24858bd4ab14759b2252ddad7f0e5cd" member_type: VOTER last_known_addr { host: "127.22.168.3" port: 35191 } } }
21:04:11.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:11.921068 23969 tablet_replica.cc:445] T 581dbe2f34ca4a7981755fa5f4b6b6ca P 1acec06b763242d082ddb75231fc4eca: This TxnStatusTablet replica's current role is: FOLLOWER
21:04:11.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:11.921653 23963 raft_consensus.cc:359] T 581dbe2f34ca4a7981755fa5f4b6b6ca P 210cdae6f0e44d83acf5641ce563b738 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "210cdae6f0e44d83acf5641ce563b738" member_type: VOTER last_known_addr { host: "127.22.168.1" port: 42241 } } peers { permanent_uuid: "1acec06b763242d082ddb75231fc4eca" member_type: VOTER last_known_addr { host: "127.22.168.2" port: 37409 } } peers { permanent_uuid: "e24858bd4ab14759b2252ddad7f0e5cd" member_type: VOTER last_known_addr { host: "127.22.168.3" port: 35191 } }
21:04:11.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:11.921854 23963 raft_consensus.cc:385] T 581dbe2f34ca4a7981755fa5f4b6b6ca P 210cdae6f0e44d83acf5641ce563b738 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
21:04:11.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:11.921939 23963 raft_consensus.cc:740] T 581dbe2f34ca4a7981755fa5f4b6b6ca P 210cdae6f0e44d83acf5641ce563b738 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 210cdae6f0e44d83acf5641ce563b738, State: Initialized, Role: FOLLOWER
21:04:11.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:11.922252 23963 consensus_queue.cc:260] T 581dbe2f34ca4a7981755fa5f4b6b6ca P 210cdae6f0e44d83acf5641ce563b738 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "210cdae6f0e44d83acf5641ce563b738" member_type: VOTER last_known_addr { host: "127.22.168.1" port: 42241 } } peers { permanent_uuid: "1acec06b763242d082ddb75231fc4eca" member_type: VOTER last_known_addr { host: "127.22.168.2" port: 37409 } } peers { permanent_uuid: "e24858bd4ab14759b2252ddad7f0e5cd" member_type: VOTER last_known_addr { host: "127.22.168.3" port: 35191 } }
21:04:11.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:11.922677 23968 tablet_replica.cc:442] T 581dbe2f34ca4a7981755fa5f4b6b6ca P 210cdae6f0e44d83acf5641ce563b738: TxnStatusTablet state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "210cdae6f0e44d83acf5641ce563b738" member_type: VOTER last_known_addr { host: "127.22.168.1" port: 42241 } } peers { permanent_uuid: "1acec06b763242d082ddb75231fc4eca" member_type: VOTER last_known_addr { host: "127.22.168.2" port: 37409 } } peers { permanent_uuid: "e24858bd4ab14759b2252ddad7f0e5cd" member_type: VOTER last_known_addr { host: "127.22.168.3" port: 35191 } } }
21:04:11.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:11.922931 23968 tablet_replica.cc:445] T 581dbe2f34ca4a7981755fa5f4b6b6ca P 210cdae6f0e44d83acf5641ce563b738: This TxnStatusTablet replica's current role is: FOLLOWER
21:04:11.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:11.926004 23963 ts_tablet_manager.cc:1434] T 581dbe2f34ca4a7981755fa5f4b6b6ca P 210cdae6f0e44d83acf5641ce563b738: Time spent starting tablet: real 0.005s user 0.002s sys 0.000s
21:04:11.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:11.926455 23962 ts_tablet_manager.cc:1434] T 581dbe2f34ca4a7981755fa5f4b6b6ca P 1acec06b763242d082ddb75231fc4eca: Time spent starting tablet: real 0.007s user 0.003s sys 0.000s
21:04:11.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:11.930450 23968 raft_consensus.cc:493] T 581dbe2f34ca4a7981755fa5f4b6b6ca P 210cdae6f0e44d83acf5641ce563b738 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
21:04:11.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:11.930655 23968 raft_consensus.cc:515] T 581dbe2f34ca4a7981755fa5f4b6b6ca P 210cdae6f0e44d83acf5641ce563b738 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "210cdae6f0e44d83acf5641ce563b738" member_type: VOTER last_known_addr { host: "127.22.168.1" port: 42241 } } peers { permanent_uuid: "1acec06b763242d082ddb75231fc4eca" member_type: VOTER last_known_addr { host: "127.22.168.2" port: 37409 } } peers { permanent_uuid: "e24858bd4ab14759b2252ddad7f0e5cd" member_type: VOTER last_known_addr { host: "127.22.168.3" port: 35191 } }
21:04:11.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:11.932521 23968 leader_election.cc:290] T 581dbe2f34ca4a7981755fa5f4b6b6ca P 210cdae6f0e44d83acf5641ce563b738 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 1acec06b763242d082ddb75231fc4eca (127.22.168.2:37409), e24858bd4ab14759b2252ddad7f0e5cd (127.22.168.3:35191)
21:04:11.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:11.936275 23964 tablet_bootstrap.cc:492] T 581dbe2f34ca4a7981755fa5f4b6b6ca P e24858bd4ab14759b2252ddad7f0e5cd: No bootstrap required, opened a new log
21:04:11.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:11.936508 23964 ts_tablet_manager.cc:1403] T 581dbe2f34ca4a7981755fa5f4b6b6ca P e24858bd4ab14759b2252ddad7f0e5cd: Time spent bootstrapping tablet: real 0.019s user 0.013s sys 0.000s
21:04:11.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:11.937412 23964 raft_consensus.cc:359] T 581dbe2f34ca4a7981755fa5f4b6b6ca P e24858bd4ab14759b2252ddad7f0e5cd [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "210cdae6f0e44d83acf5641ce563b738" member_type: VOTER last_known_addr { host: "127.22.168.1" port: 42241 } } peers { permanent_uuid: "1acec06b763242d082ddb75231fc4eca" member_type: VOTER last_known_addr { host: "127.22.168.2" port: 37409 } } peers { permanent_uuid: "e24858bd4ab14759b2252ddad7f0e5cd" member_type: VOTER last_known_addr { host: "127.22.168.3" port: 35191 } }
21:04:11.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:11.937626 23964 raft_consensus.cc:385] T 581dbe2f34ca4a7981755fa5f4b6b6ca P e24858bd4ab14759b2252ddad7f0e5cd [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
21:04:11.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:11.937716 23964 raft_consensus.cc:740] T 581dbe2f34ca4a7981755fa5f4b6b6ca P e24858bd4ab14759b2252ddad7f0e5cd [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: e24858bd4ab14759b2252ddad7f0e5cd, State: Initialized, Role: FOLLOWER
21:04:11.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:11.938062 23964 consensus_queue.cc:260] T 581dbe2f34ca4a7981755fa5f4b6b6ca P e24858bd4ab14759b2252ddad7f0e5cd [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "210cdae6f0e44d83acf5641ce563b738" member_type: VOTER last_known_addr { host: "127.22.168.1" port: 42241 } } peers { permanent_uuid: "1acec06b763242d082ddb75231fc4eca" member_type: VOTER last_known_addr { host: "127.22.168.2" port: 37409 } } peers { permanent_uuid: "e24858bd4ab14759b2252ddad7f0e5cd" member_type: VOTER last_known_addr { host: "127.22.168.3" port: 35191 } }
21:04:11.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:11.938519 23970 tablet_replica.cc:442] T 581dbe2f34ca4a7981755fa5f4b6b6ca P e24858bd4ab14759b2252ddad7f0e5cd: TxnStatusTablet state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "210cdae6f0e44d83acf5641ce563b738" member_type: VOTER last_known_addr { host: "127.22.168.1" port: 42241 } } peers { permanent_uuid: "1acec06b763242d082ddb75231fc4eca" member_type: VOTER last_known_addr { host: "127.22.168.2" port: 37409 } } peers { permanent_uuid: "e24858bd4ab14759b2252ddad7f0e5cd" member_type: VOTER last_known_addr { host: "127.22.168.3" port: 35191 } } }
21:04:11.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:11.938758 23970 tablet_replica.cc:445] T 581dbe2f34ca4a7981755fa5f4b6b6ca P e24858bd4ab14759b2252ddad7f0e5cd: This TxnStatusTablet replica's current role is: FOLLOWER
21:04:11.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:11.940702 23964 ts_tablet_manager.cc:1434] T 581dbe2f34ca4a7981755fa5f4b6b6ca P e24858bd4ab14759b2252ddad7f0e5cd: Time spent starting tablet: real 0.004s user 0.002s sys 0.000s
21:04:11.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:11.941427 23749 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "581dbe2f34ca4a7981755fa5f4b6b6ca" candidate_uuid: "210cdae6f0e44d83acf5641ce563b738" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "1acec06b763242d082ddb75231fc4eca" is_pre_election: true
21:04:11.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:11.941671 23749 raft_consensus.cc:2468] T 581dbe2f34ca4a7981755fa5f4b6b6ca P 1acec06b763242d082ddb75231fc4eca [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 210cdae6f0e44d83acf5641ce563b738 in term 0.
21:04:11.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:11.942202 23531 leader_election.cc:304] T 581dbe2f34ca4a7981755fa5f4b6b6ca P 210cdae6f0e44d83acf5641ce563b738 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 1acec06b763242d082ddb75231fc4eca, 210cdae6f0e44d83acf5641ce563b738; no voters:
21:04:11.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:11.942633 23968 raft_consensus.cc:2804] T 581dbe2f34ca4a7981755fa5f4b6b6ca P 210cdae6f0e44d83acf5641ce563b738 [term 0 FOLLOWER]: Leader pre-election won for term 1
21:04:11.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:11.942756 23968 raft_consensus.cc:493] T 581dbe2f34ca4a7981755fa5f4b6b6ca P 210cdae6f0e44d83acf5641ce563b738 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
21:04:11.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:11.942858 23968 raft_consensus.cc:3060] T 581dbe2f34ca4a7981755fa5f4b6b6ca P 210cdae6f0e44d83acf5641ce563b738 [term 0 FOLLOWER]: Advancing to term 1
21:04:11.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:11.945075 23891 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "581dbe2f34ca4a7981755fa5f4b6b6ca" candidate_uuid: "210cdae6f0e44d83acf5641ce563b738" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "e24858bd4ab14759b2252ddad7f0e5cd" is_pre_election: true
21:04:11.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:11.945315 23891 raft_consensus.cc:2468] T 581dbe2f34ca4a7981755fa5f4b6b6ca P e24858bd4ab14759b2252ddad7f0e5cd [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 210cdae6f0e44d83acf5641ce563b738 in term 0.
21:04:11.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:11.945919 23968 raft_consensus.cc:515] T 581dbe2f34ca4a7981755fa5f4b6b6ca P 210cdae6f0e44d83acf5641ce563b738 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "210cdae6f0e44d83acf5641ce563b738" member_type: VOTER last_known_addr { host: "127.22.168.1" port: 42241 } } peers { permanent_uuid: "1acec06b763242d082ddb75231fc4eca" member_type: VOTER last_known_addr { host: "127.22.168.2" port: 37409 } } peers { permanent_uuid: "e24858bd4ab14759b2252ddad7f0e5cd" member_type: VOTER last_known_addr { host: "127.22.168.3" port: 35191 } }
21:04:11.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:11.946938 23749 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "581dbe2f34ca4a7981755fa5f4b6b6ca" candidate_uuid: "210cdae6f0e44d83acf5641ce563b738" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "1acec06b763242d082ddb75231fc4eca"
21:04:11.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:11.947177 23749 raft_consensus.cc:3060] T 581dbe2f34ca4a7981755fa5f4b6b6ca P 1acec06b763242d082ddb75231fc4eca [term 0 FOLLOWER]: Advancing to term 1
21:04:11.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:11.947252 23891 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "581dbe2f34ca4a7981755fa5f4b6b6ca" candidate_uuid: "210cdae6f0e44d83acf5641ce563b738" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "e24858bd4ab14759b2252ddad7f0e5cd"
21:04:11.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:11.947440 23891 raft_consensus.cc:3060] T 581dbe2f34ca4a7981755fa5f4b6b6ca P e24858bd4ab14759b2252ddad7f0e5cd [term 0 FOLLOWER]: Advancing to term 1
21:04:11.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:11.949510 23749 raft_consensus.cc:2468] T 581dbe2f34ca4a7981755fa5f4b6b6ca P 1acec06b763242d082ddb75231fc4eca [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 210cdae6f0e44d83acf5641ce563b738 in term 1.
21:04:11.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:11.949841 23891 raft_consensus.cc:2468] T 581dbe2f34ca4a7981755fa5f4b6b6ca P e24858bd4ab14759b2252ddad7f0e5cd [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 210cdae6f0e44d83acf5641ce563b738 in term 1.
21:04:11.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:11.949916 23968 leader_election.cc:290] T 581dbe2f34ca4a7981755fa5f4b6b6ca P 210cdae6f0e44d83acf5641ce563b738 [CANDIDATE]: Term 1 election: Requested vote from peers 1acec06b763242d082ddb75231fc4eca (127.22.168.2:37409), e24858bd4ab14759b2252ddad7f0e5cd (127.22.168.3:35191)
21:04:11.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:11.949945 23531 leader_election.cc:304] T 581dbe2f34ca4a7981755fa5f4b6b6ca P 210cdae6f0e44d83acf5641ce563b738 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 1acec06b763242d082ddb75231fc4eca, 210cdae6f0e44d83acf5641ce563b738; no voters:
21:04:11.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:11.950248 23968 raft_consensus.cc:2804] T 581dbe2f34ca4a7981755fa5f4b6b6ca P 210cdae6f0e44d83acf5641ce563b738 [term 1 FOLLOWER]: Leader election won for term 1
21:04:11.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:11.950712 23968 raft_consensus.cc:697] T 581dbe2f34ca4a7981755fa5f4b6b6ca P 210cdae6f0e44d83acf5641ce563b738 [term 1 LEADER]: Becoming Leader. State: Replica: 210cdae6f0e44d83acf5641ce563b738, State: Running, Role: LEADER
21:04:11.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:11.951562 23968 consensus_queue.cc:237] T 581dbe2f34ca4a7981755fa5f4b6b6ca P 210cdae6f0e44d83acf5641ce563b738 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "210cdae6f0e44d83acf5641ce563b738" member_type: VOTER last_known_addr { host: "127.22.168.1" port: 42241 } } peers { permanent_uuid: "1acec06b763242d082ddb75231fc4eca" member_type: VOTER last_known_addr { host: "127.22.168.2" port: 37409 } } peers { permanent_uuid: "e24858bd4ab14759b2252ddad7f0e5cd" member_type: VOTER last_known_addr { host: "127.22.168.3" port: 35191 } }
21:04:11.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:11.955256 23995 tablet_replica.cc:442] T 581dbe2f34ca4a7981755fa5f4b6b6ca P 210cdae6f0e44d83acf5641ce563b738: TxnStatusTablet state changed. Reason: New leader 210cdae6f0e44d83acf5641ce563b738. Latest consensus state: current_term: 1 leader_uuid: "210cdae6f0e44d83acf5641ce563b738" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "210cdae6f0e44d83acf5641ce563b738" member_type: VOTER last_known_addr { host: "127.22.168.1" port: 42241 } } peers { permanent_uuid: "1acec06b763242d082ddb75231fc4eca" member_type: VOTER last_known_addr { host: "127.22.168.2" port: 37409 } } peers { permanent_uuid: "e24858bd4ab14759b2252ddad7f0e5cd" member_type: VOTER last_known_addr { host: "127.22.168.3" port: 35191 } } }
21:04:11.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:11.955643 23995 tablet_replica.cc:445] T 581dbe2f34ca4a7981755fa5f4b6b6ca P 210cdae6f0e44d83acf5641ce563b738: This TxnStatusTablet replica's current role is: LEADER
21:04:11.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:11.956712 23997 txn_status_manager.cc:874] Waiting until node catch up with all replicated operations in previous term...
21:04:11.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:11.957034 23997 txn_status_manager.cc:930] Loading transaction status metadata into memory...
21:04:11.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:11.958292 23404 catalog_manager.cc:5649] T 581dbe2f34ca4a7981755fa5f4b6b6ca P 210cdae6f0e44d83acf5641ce563b738 reported cstate change: term changed from 0 to 1, leader changed from <none> to 210cdae6f0e44d83acf5641ce563b738 (127.22.168.1). New cstate: current_term: 1 leader_uuid: "210cdae6f0e44d83acf5641ce563b738" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "210cdae6f0e44d83acf5641ce563b738" member_type: VOTER last_known_addr { host: "127.22.168.1" port: 42241 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "1acec06b763242d082ddb75231fc4eca" member_type: VOTER last_known_addr { host: "127.22.168.2" port: 37409 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "e24858bd4ab14759b2252ddad7f0e5cd" member_type: VOTER last_known_addr { host: "127.22.168.3" port: 35191 } health_report { overall_health: UNKNOWN } } }
21:04:11.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:11.997452 23891 raft_consensus.cc:1275] T 581dbe2f34ca4a7981755fa5f4b6b6ca P e24858bd4ab14759b2252ddad7f0e5cd [term 1 FOLLOWER]: Refusing update from remote peer 210cdae6f0e44d83acf5641ce563b738: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
21:04:11.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:11.997498 23749 raft_consensus.cc:1275] T 581dbe2f34ca4a7981755fa5f4b6b6ca P 1acec06b763242d082ddb75231fc4eca [term 1 FOLLOWER]: Refusing update from remote peer 210cdae6f0e44d83acf5641ce563b738: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
21:04:11.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:11.998440 23995 consensus_queue.cc:1048] T 581dbe2f34ca4a7981755fa5f4b6b6ca P 210cdae6f0e44d83acf5641ce563b738 [LEADER]: Connected to new peer: Peer: permanent_uuid: "1acec06b763242d082ddb75231fc4eca" member_type: VOTER last_known_addr { host: "127.22.168.2" port: 37409 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
21:04:11.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:11.998696 23968 consensus_queue.cc:1048] T 581dbe2f34ca4a7981755fa5f4b6b6ca P 210cdae6f0e44d83acf5641ce563b738 [LEADER]: Connected to new peer: Peer: permanent_uuid: "e24858bd4ab14759b2252ddad7f0e5cd" member_type: VOTER last_known_addr { host: "127.22.168.3" port: 35191 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
21:04:12.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:12.014397 23970 tablet_replica.cc:442] T 581dbe2f34ca4a7981755fa5f4b6b6ca P e24858bd4ab14759b2252ddad7f0e5cd: TxnStatusTablet state changed. Reason: New leader 210cdae6f0e44d83acf5641ce563b738. Latest consensus state: current_term: 1 leader_uuid: "210cdae6f0e44d83acf5641ce563b738" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "210cdae6f0e44d83acf5641ce563b738" member_type: VOTER last_known_addr { host: "127.22.168.1" port: 42241 } } peers { permanent_uuid: "1acec06b763242d082ddb75231fc4eca" member_type: VOTER last_known_addr { host: "127.22.168.2" port: 37409 } } peers { permanent_uuid: "e24858bd4ab14759b2252ddad7f0e5cd" member_type: VOTER last_known_addr { host: "127.22.168.3" port: 35191 } } }
21:04:12.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:12.014664 23970 tablet_replica.cc:445] T 581dbe2f34ca4a7981755fa5f4b6b6ca P e24858bd4ab14759b2252ddad7f0e5cd: This TxnStatusTablet replica's current role is: FOLLOWER
21:04:12.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:12.017900 24000 mvcc.cc:204] Tried to move back new op lower bound from 7215863611371954176 to 7215863611197964288. Current Snapshot: MvccSnapshot[applied={T|T < 7215863611371954176}]
21:04:12.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:12.018507 24001 mvcc.cc:204] Tried to move back new op lower bound from 7215863611371954176 to 7215863611197964288. Current Snapshot: MvccSnapshot[applied={T|T < 7215863611371954176}]
21:04:12.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:12.019743 23970 tablet_replica.cc:442] T 581dbe2f34ca4a7981755fa5f4b6b6ca P e24858bd4ab14759b2252ddad7f0e5cd: TxnStatusTablet state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "210cdae6f0e44d83acf5641ce563b738" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "210cdae6f0e44d83acf5641ce563b738" member_type: VOTER last_known_addr { host: "127.22.168.1" port: 42241 } } peers { permanent_uuid: "1acec06b763242d082ddb75231fc4eca" member_type: VOTER last_known_addr { host: "127.22.168.2" port: 37409 } } peers { permanent_uuid: "e24858bd4ab14759b2252ddad7f0e5cd" member_type: VOTER last_known_addr { host: "127.22.168.3" port: 35191 } } }
21:04:12.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:12.019968 23970 tablet_replica.cc:445] T 581dbe2f34ca4a7981755fa5f4b6b6ca P e24858bd4ab14759b2252ddad7f0e5cd: This TxnStatusTablet replica's current role is: FOLLOWER
21:04:12.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:12.022315 23968 tablet_replica.cc:442] T 581dbe2f34ca4a7981755fa5f4b6b6ca P 210cdae6f0e44d83acf5641ce563b738: TxnStatusTablet state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "210cdae6f0e44d83acf5641ce563b738" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "210cdae6f0e44d83acf5641ce563b738" member_type: VOTER last_known_addr { host: "127.22.168.1" port: 42241 } } peers { permanent_uuid: "1acec06b763242d082ddb75231fc4eca" member_type: VOTER last_known_addr { host: "127.22.168.2" port: 37409 } } peers { permanent_uuid: "e24858bd4ab14759b2252ddad7f0e5cd" member_type: VOTER last_known_addr { host: "127.22.168.3" port: 35191 } } }
21:04:12.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:12.022563 23968 tablet_replica.cc:445] T 581dbe2f34ca4a7981755fa5f4b6b6ca P 210cdae6f0e44d83acf5641ce563b738: This TxnStatusTablet replica's current role is: LEADER
21:04:12.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:12.025528 23969 tablet_replica.cc:442] T 581dbe2f34ca4a7981755fa5f4b6b6ca P 1acec06b763242d082ddb75231fc4eca: TxnStatusTablet state changed. Reason: New leader 210cdae6f0e44d83acf5641ce563b738. Latest consensus state: current_term: 1 leader_uuid: "210cdae6f0e44d83acf5641ce563b738" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "210cdae6f0e44d83acf5641ce563b738" member_type: VOTER last_known_addr { host: "127.22.168.1" port: 42241 } } peers { permanent_uuid: "1acec06b763242d082ddb75231fc4eca" member_type: VOTER last_known_addr { host: "127.22.168.2" port: 37409 } } peers { permanent_uuid: "e24858bd4ab14759b2252ddad7f0e5cd" member_type: VOTER last_known_addr { host: "127.22.168.3" port: 35191 } } }
21:04:12.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:12.025759 23969 tablet_replica.cc:445] T 581dbe2f34ca4a7981755fa5f4b6b6ca P 1acec06b763242d082ddb75231fc4eca: This TxnStatusTablet replica's current role is: FOLLOWER
21:04:12.029 [DEBUG - Time-limited test] (KuduTransaction.java:702) starting keepalive heartbeating with period 500 ms (txn ID 0)
21:04:12.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:12.032681 23969 tablet_replica.cc:442] T 581dbe2f34ca4a7981755fa5f4b6b6ca P 1acec06b763242d082ddb75231fc4eca: TxnStatusTablet state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "210cdae6f0e44d83acf5641ce563b738" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "210cdae6f0e44d83acf5641ce563b738" member_type: VOTER last_known_addr { host: "127.22.168.1" port: 42241 } } peers { permanent_uuid: "1acec06b763242d082ddb75231fc4eca" member_type: VOTER last_known_addr { host: "127.22.168.2" port: 37409 } } peers { permanent_uuid: "e24858bd4ab14759b2252ddad7f0e5cd" member_type: VOTER last_known_addr { host: "127.22.168.3" port: 35191 } } }
21:04:12.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:12.032912 23969 tablet_replica.cc:445] T 581dbe2f34ca4a7981755fa5f4b6b6ca P 1acec06b763242d082ddb75231fc4eca: This TxnStatusTablet replica's current role is: FOLLOWER
21:04:12.038 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:894) Opened table 03fcdef52a6f4a7099b05a3d2fa91c76
21:04:12.038 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.22.168.60:33247
21:04:12.039 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 2
}
}
21:04:12.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:12.037994 23968 tablet_replica.cc:442] T 581dbe2f34ca4a7981755fa5f4b6b6ca P 210cdae6f0e44d83acf5641ce563b738: TxnStatusTablet state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "210cdae6f0e44d83acf5641ce563b738" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "210cdae6f0e44d83acf5641ce563b738" member_type: VOTER last_known_addr { host: "127.22.168.1" port: 42241 } } peers { permanent_uuid: "1acec06b763242d082ddb75231fc4eca" member_type: VOTER last_known_addr { host: "127.22.168.2" port: 37409 } } peers { permanent_uuid: "e24858bd4ab14759b2252ddad7f0e5cd" member_type: VOTER last_known_addr { host: "127.22.168.3" port: 35191 } } }
21:04:12.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:12.038245 23968 tablet_replica.cc:445] T 581dbe2f34ca4a7981755fa5f4b6b6ca P 210cdae6f0e44d83acf5641ce563b738: This TxnStatusTablet replica's current role is: LEADER
21:04:12.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:12.040283 23200 external_mini_cluster.cc:1699] Attempting to check leaks for /tmp/dist-test-taskRxIrUU/build/asan/bin/kudu pid 23371
21:04:12.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:12.044799 23444 generic_service.cc:196] Checking for leaks (request via RPC)
21:04:12.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:12.111339 23969 consensus_queue.cc:1048] T 260c684b258041bab84c3ff36578f96b P 1acec06b763242d082ddb75231fc4eca [LEADER]: Connected to new peer: Peer: permanent_uuid: "210cdae6f0e44d83acf5641ce563b738" member_type: VOTER last_known_addr { host: "127.22.168.1" port: 42241 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
21:04:12.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:12.121201 23969 consensus_queue.cc:1048] T 260c684b258041bab84c3ff36578f96b P 1acec06b763242d082ddb75231fc4eca [LEADER]: Connected to new peer: Peer: permanent_uuid: "e24858bd4ab14759b2252ddad7f0e5cd" member_type: VOTER last_known_addr { host: "127.22.168.3" port: 35191 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
21:04:12.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:12.175288 23200 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskRxIrUU/build/asan/bin/kudu with pid 23371
21:04:12.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:12.205848 23818 connection.cc:537] client connection to 127.22.168.60:33247 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
21:04:12.206 [DEBUG - kudu-nio-1] (Connection.java:239) [peer master-127.22.168.60:33247(127.22.168.60:33247)] handling channelInactive
21:04:12.206 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.60:33247(127.22.168.60:33247)] cleaning up while in state READY due to: connection closed
21:04:12.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:12.206516 23936 heartbeater.cc:646] Failed to heartbeat to 127.22.168.60:33247 (0 consecutive failures): Network error: Failed to send heartbeat to master: recv error from unknown peer: Transport endpoint is not connected (error 107)
21:04:12.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:12.206924 23794 heartbeater.cc:646] Failed to heartbeat to 127.22.168.60:33247 (0 consecutive failures): Network error: Failed to send heartbeat to master: recv got EOF from 127.22.168.60:33247 (error 108)
21:04:12.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:12.207250 23532 connection.cc:537] client connection to 127.22.168.60:33247 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
21:04:12.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:12.207928 23653 heartbeater.cc:646] Failed to heartbeat to 127.22.168.60:33247 (0 consecutive failures): Network error: Failed to send heartbeat to master: recv error from unknown peer: Transport endpoint is not connected (error 107)
21:04:12.208 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
21:04:12.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:12.250986 23970 consensus_queue.cc:1048] T 01aa0e192c434b468ec7ed90d48563b3 P e24858bd4ab14759b2252ddad7f0e5cd [LEADER]: Connected to new peer: Peer: permanent_uuid: "210cdae6f0e44d83acf5641ce563b738" member_type: VOTER last_known_addr { host: "127.22.168.1" port: 42241 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
21:04:12.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:12.259363 23970 consensus_queue.cc:1048] T 01aa0e192c434b468ec7ed90d48563b3 P e24858bd4ab14759b2252ddad7f0e5cd [LEADER]: Connected to new peer: Peer: permanent_uuid: "1acec06b763242d082ddb75231fc4eca" member_type: VOTER last_known_addr { host: "127.22.168.2" port: 37409 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
21:04:12.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:12.474831 24019 raft_consensus.cc:493] T 00000000000000000000000000000000 P c520077acf3341e29447c6c2b8d3f74c [term 1 FOLLOWER]: Starting pre-election (detected failure of leader eb664add87f44013b596f4cc01c9a41c)
21:04:12.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:12.475080 24019 raft_consensus.cc:515] T 00000000000000000000000000000000 P c520077acf3341e29447c6c2b8d3f74c [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e9dd218ca2244f5a934aac2b7e58eb1d" member_type: VOTER last_known_addr { host: "127.22.168.62" port: 37239 } } peers { permanent_uuid: "c520077acf3341e29447c6c2b8d3f74c" member_type: VOTER last_known_addr { host: "127.22.168.61" port: 32947 } } peers { permanent_uuid: "eb664add87f44013b596f4cc01c9a41c" member_type: VOTER last_known_addr { host: "127.22.168.60" port: 33247 } }
21:04:12.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:12.475597 24020 raft_consensus.cc:493] T 00000000000000000000000000000000 P e9dd218ca2244f5a934aac2b7e58eb1d [term 1 FOLLOWER]: Starting pre-election (detected failure of leader eb664add87f44013b596f4cc01c9a41c)
21:04:12.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:12.475788 24020 raft_consensus.cc:515] T 00000000000000000000000000000000 P e9dd218ca2244f5a934aac2b7e58eb1d [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e9dd218ca2244f5a934aac2b7e58eb1d" member_type: VOTER last_known_addr { host: "127.22.168.62" port: 37239 } } peers { permanent_uuid: "c520077acf3341e29447c6c2b8d3f74c" member_type: VOTER last_known_addr { host: "127.22.168.61" port: 32947 } } peers { permanent_uuid: "eb664add87f44013b596f4cc01c9a41c" member_type: VOTER last_known_addr { host: "127.22.168.60" port: 33247 } }
21:04:12.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:12.476668 24019 leader_election.cc:290] T 00000000000000000000000000000000 P c520077acf3341e29447c6c2b8d3f74c [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers e9dd218ca2244f5a934aac2b7e58eb1d (127.22.168.62:37239), eb664add87f44013b596f4cc01c9a41c (127.22.168.60:33247)
21:04:12.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:12.476967 24020 leader_election.cc:290] T 00000000000000000000000000000000 P e9dd218ca2244f5a934aac2b7e58eb1d [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers c520077acf3341e29447c6c2b8d3f74c (127.22.168.61:32947), eb664add87f44013b596f4cc01c9a41c (127.22.168.60:33247)
21:04:12.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:12.477200 23251 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "c520077acf3341e29447c6c2b8d3f74c" candidate_term: 2 candidate_status { last_received { term: 1 index: 11 } } ignore_live_leader: false dest_uuid: "e9dd218ca2244f5a934aac2b7e58eb1d" is_pre_election: true
21:04:12.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:12.477406 23251 raft_consensus.cc:2468] T 00000000000000000000000000000000 P e9dd218ca2244f5a934aac2b7e58eb1d [term 1 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate c520077acf3341e29447c6c2b8d3f74c in term 1.
21:04:12.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:12.477715 23331 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "e9dd218ca2244f5a934aac2b7e58eb1d" candidate_term: 2 candidate_status { last_received { term: 1 index: 11 } } ignore_live_leader: false dest_uuid: "c520077acf3341e29447c6c2b8d3f74c" is_pre_election: true
21:04:12.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:12.477967 23331 raft_consensus.cc:2468] T 00000000000000000000000000000000 P c520077acf3341e29447c6c2b8d3f74c [term 1 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate e9dd218ca2244f5a934aac2b7e58eb1d in term 1.
21:04:12.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:12.478592 23307 leader_election.cc:304] T 00000000000000000000000000000000 P c520077acf3341e29447c6c2b8d3f74c [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: c520077acf3341e29447c6c2b8d3f74c, e9dd218ca2244f5a934aac2b7e58eb1d; no voters:
21:04:12.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:12.478976 24019 raft_consensus.cc:2804] T 00000000000000000000000000000000 P c520077acf3341e29447c6c2b8d3f74c [term 1 FOLLOWER]: Leader pre-election won for term 2
21:04:12.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:12.478964 23229 leader_election.cc:304] T 00000000000000000000000000000000 P e9dd218ca2244f5a934aac2b7e58eb1d [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: c520077acf3341e29447c6c2b8d3f74c, e9dd218ca2244f5a934aac2b7e58eb1d; no voters:
21:04:12.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:12.479094 24019 raft_consensus.cc:493] T 00000000000000000000000000000000 P c520077acf3341e29447c6c2b8d3f74c [term 1 FOLLOWER]: Starting leader election (detected failure of leader eb664add87f44013b596f4cc01c9a41c)
21:04:12.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:12.479218 24019 raft_consensus.cc:3060] T 00000000000000000000000000000000 P c520077acf3341e29447c6c2b8d3f74c [term 1 FOLLOWER]: Advancing to term 2
21:04:12.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:12.479347 24020 raft_consensus.cc:2804] T 00000000000000000000000000000000 P e9dd218ca2244f5a934aac2b7e58eb1d [term 1 FOLLOWER]: Leader pre-election won for term 2
21:04:12.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:12.479508 23310 leader_election.cc:336] T 00000000000000000000000000000000 P c520077acf3341e29447c6c2b8d3f74c [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer eb664add87f44013b596f4cc01c9a41c (127.22.168.60:33247): Network error: Client connection negotiation failed: client connection to 127.22.168.60:33247: connect: Connection refused (error 111)
21:04:12.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:12.479480 24020 raft_consensus.cc:493] T 00000000000000000000000000000000 P e9dd218ca2244f5a934aac2b7e58eb1d [term 1 FOLLOWER]: Starting leader election (detected failure of leader eb664add87f44013b596f4cc01c9a41c)
21:04:12.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:12.479678 24020 raft_consensus.cc:3060] T 00000000000000000000000000000000 P e9dd218ca2244f5a934aac2b7e58eb1d [term 1 FOLLOWER]: Advancing to term 2
21:04:12.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:12.481099 23229 leader_election.cc:336] T 00000000000000000000000000000000 P e9dd218ca2244f5a934aac2b7e58eb1d [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer eb664add87f44013b596f4cc01c9a41c (127.22.168.60:33247): Network error: Client connection negotiation failed: client connection to 127.22.168.60:33247: connect: Connection refused (error 111)
21:04:12.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:12.481719 24019 raft_consensus.cc:515] T 00000000000000000000000000000000 P c520077acf3341e29447c6c2b8d3f74c [term 2 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e9dd218ca2244f5a934aac2b7e58eb1d" member_type: VOTER last_known_addr { host: "127.22.168.62" port: 37239 } } peers { permanent_uuid: "c520077acf3341e29447c6c2b8d3f74c" member_type: VOTER last_known_addr { host: "127.22.168.61" port: 32947 } } peers { permanent_uuid: "eb664add87f44013b596f4cc01c9a41c" member_type: VOTER last_known_addr { host: "127.22.168.60" port: 33247 } }
21:04:12.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:12.482115 24020 raft_consensus.cc:515] T 00000000000000000000000000000000 P e9dd218ca2244f5a934aac2b7e58eb1d [term 2 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e9dd218ca2244f5a934aac2b7e58eb1d" member_type: VOTER last_known_addr { host: "127.22.168.62" port: 37239 } } peers { permanent_uuid: "c520077acf3341e29447c6c2b8d3f74c" member_type: VOTER last_known_addr { host: "127.22.168.61" port: 32947 } } peers { permanent_uuid: "eb664add87f44013b596f4cc01c9a41c" member_type: VOTER last_known_addr { host: "127.22.168.60" port: 33247 } }
21:04:12.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:12.482575 24019 leader_election.cc:290] T 00000000000000000000000000000000 P c520077acf3341e29447c6c2b8d3f74c [CANDIDATE]: Term 2 election: Requested vote from peers e9dd218ca2244f5a934aac2b7e58eb1d (127.22.168.62:37239), eb664add87f44013b596f4cc01c9a41c (127.22.168.60:33247)
21:04:12.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:12.482724 24020 leader_election.cc:290] T 00000000000000000000000000000000 P e9dd218ca2244f5a934aac2b7e58eb1d [CANDIDATE]: Term 2 election: Requested vote from peers c520077acf3341e29447c6c2b8d3f74c (127.22.168.61:32947), eb664add87f44013b596f4cc01c9a41c (127.22.168.60:33247)
21:04:12.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:12.482669 23251 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "c520077acf3341e29447c6c2b8d3f74c" candidate_term: 2 candidate_status { last_received { term: 1 index: 11 } } ignore_live_leader: false dest_uuid: "e9dd218ca2244f5a934aac2b7e58eb1d"
21:04:12.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:12.482918 23251 raft_consensus.cc:2393] T 00000000000000000000000000000000 P e9dd218ca2244f5a934aac2b7e58eb1d [term 2 FOLLOWER]: Leader election vote request: Denying vote to candidate c520077acf3341e29447c6c2b8d3f74c in current term 2: Already voted for candidate e9dd218ca2244f5a934aac2b7e58eb1d in this term.
21:04:12.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:12.482993 23331 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "e9dd218ca2244f5a934aac2b7e58eb1d" candidate_term: 2 candidate_status { last_received { term: 1 index: 11 } } ignore_live_leader: false dest_uuid: "c520077acf3341e29447c6c2b8d3f74c"
21:04:12.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:12.483285 23331 raft_consensus.cc:2393] T 00000000000000000000000000000000 P c520077acf3341e29447c6c2b8d3f74c [term 2 FOLLOWER]: Leader election vote request: Denying vote to candidate e9dd218ca2244f5a934aac2b7e58eb1d in current term 2: Already voted for candidate c520077acf3341e29447c6c2b8d3f74c in this term.
21:04:12.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:12.483934 23310 leader_election.cc:336] T 00000000000000000000000000000000 P c520077acf3341e29447c6c2b8d3f74c [CANDIDATE]: Term 2 election: RPC error from VoteRequest() call to peer eb664add87f44013b596f4cc01c9a41c (127.22.168.60:33247): Network error: Client connection negotiation failed: client connection to 127.22.168.60:33247: connect: Connection refused (error 111)
21:04:12.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:12.484195 23307 leader_election.cc:304] T 00000000000000000000000000000000 P c520077acf3341e29447c6c2b8d3f74c [CANDIDATE]: Term 2 election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: c520077acf3341e29447c6c2b8d3f74c; no voters: e9dd218ca2244f5a934aac2b7e58eb1d, eb664add87f44013b596f4cc01c9a41c
21:04:12.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:12.484607 24019 raft_consensus.cc:2749] T 00000000000000000000000000000000 P c520077acf3341e29447c6c2b8d3f74c [term 2 FOLLOWER]: Leader election lost for term 2. Reason: could not achieve majority
21:04:12.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:12.484598 23229 leader_election.cc:336] T 00000000000000000000000000000000 P e9dd218ca2244f5a934aac2b7e58eb1d [CANDIDATE]: Term 2 election: RPC error from VoteRequest() call to peer eb664add87f44013b596f4cc01c9a41c (127.22.168.60:33247): Network error: Client connection negotiation failed: client connection to 127.22.168.60:33247: connect: Connection refused (error 111)
21:04:12.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:12.484730 23229 leader_election.cc:304] T 00000000000000000000000000000000 P e9dd218ca2244f5a934aac2b7e58eb1d [CANDIDATE]: Term 2 election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: e9dd218ca2244f5a934aac2b7e58eb1d; no voters: c520077acf3341e29447c6c2b8d3f74c, eb664add87f44013b596f4cc01c9a41c
21:04:12.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:12.484998 24020 raft_consensus.cc:2749] T 00000000000000000000000000000000 P e9dd218ca2244f5a934aac2b7e58eb1d [term 2 FOLLOWER]: Leader election lost for term 2. Reason: could not achieve majority
21:04:12.544 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.60:33247(127.22.168.60:33247)] connecting to peer
21:04:12.546 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.60:33247(127.22.168.60:33247): Connection refused: /127.22.168.60:33247
21:04:12.546 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.60:33247(127.22.168.60:33247)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.60:33247(127.22.168.60:33247): Connection refused: /127.22.168.60:33247
21:04:12.546 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location master-127.22.168.60:33247(127.22.168.60:33247) for tablet Kudu Master: Failed to connect to peer master-127.22.168.60:33247(127.22.168.60:33247): Connection refused: /127.22.168.60:33247
21:04:12.564 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.60:33247(127.22.168.60:33247)] connecting to peer
21:04:12.565 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.60:33247(127.22.168.60:33247): Connection refused: /127.22.168.60:33247
21:04:12.566 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.60:33247(127.22.168.60:33247)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.60:33247(127.22.168.60:33247): Connection refused: /127.22.168.60:33247
21:04:12.566 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.60:33247: Failed to connect to peer master-127.22.168.60:33247(127.22.168.60:33247): Connection refused: /127.22.168.60:33247
21:04:12.567 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:12.584 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.60:33247(127.22.168.60:33247)] connecting to peer
21:04:12.585 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.60:33247(127.22.168.60:33247): Connection refused: /127.22.168.60:33247
21:04:12.585 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.60:33247(127.22.168.60:33247)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.60:33247(127.22.168.60:33247): Connection refused: /127.22.168.60:33247
21:04:12.586 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.60:33247: Failed to connect to peer master-127.22.168.60:33247(127.22.168.60:33247): Connection refused: /127.22.168.60:33247
21:04:12.586 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:12.604 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.60:33247(127.22.168.60:33247)] connecting to peer
21:04:12.606 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.60:33247(127.22.168.60:33247): Connection refused: /127.22.168.60:33247
21:04:12.606 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.60:33247(127.22.168.60:33247)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.60:33247(127.22.168.60:33247): Connection refused: /127.22.168.60:33247
21:04:12.606 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.60:33247: Failed to connect to peer master-127.22.168.60:33247(127.22.168.60:33247): Connection refused: /127.22.168.60:33247
21:04:12.606 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:12.624 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.60:33247(127.22.168.60:33247)] connecting to peer
21:04:12.626 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.60:33247(127.22.168.60:33247): Connection refused: /127.22.168.60:33247
21:04:12.626 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.60:33247(127.22.168.60:33247)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.60:33247(127.22.168.60:33247): Connection refused: /127.22.168.60:33247
21:04:12.626 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.60:33247: Failed to connect to peer master-127.22.168.60:33247(127.22.168.60:33247): Connection refused: /127.22.168.60:33247
21:04:12.627 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:12.645 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.60:33247(127.22.168.60:33247)] connecting to peer
21:04:12.646 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.60:33247(127.22.168.60:33247): Connection refused: /127.22.168.60:33247
21:04:12.646 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.60:33247(127.22.168.60:33247)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.60:33247(127.22.168.60:33247): Connection refused: /127.22.168.60:33247
21:04:12.646 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.60:33247: Failed to connect to peer master-127.22.168.60:33247(127.22.168.60:33247): Connection refused: /127.22.168.60:33247
21:04:12.647 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:12.704 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.60:33247(127.22.168.60:33247)] connecting to peer
21:04:12.705 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.60:33247(127.22.168.60:33247): Connection refused: /127.22.168.60:33247
21:04:12.705 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.60:33247(127.22.168.60:33247)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.60:33247(127.22.168.60:33247): Connection refused: /127.22.168.60:33247
21:04:12.705 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.60:33247: Failed to connect to peer master-127.22.168.60:33247(127.22.168.60:33247): Connection refused: /127.22.168.60:33247
21:04:12.706 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:12.764 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.60:33247(127.22.168.60:33247)] connecting to peer
21:04:12.765 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.60:33247(127.22.168.60:33247): Connection refused: /127.22.168.60:33247
21:04:12.765 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.60:33247(127.22.168.60:33247)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.60:33247(127.22.168.60:33247): Connection refused: /127.22.168.60:33247
21:04:12.766 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.60:33247: Failed to connect to peer master-127.22.168.60:33247(127.22.168.60:33247): Connection refused: /127.22.168.60:33247
21:04:12.766 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:12.767 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=Kudu Master, attempt=8, TimeoutTracker(timeout=250, elapsed=222), Traces: [0ms] sending RPC to server master-127.22.168.60:33247, [2ms] received response from server master-127.22.168.60:33247: Network error: Failed to connect to peer master-127.22.168.60:33247(127.22.168.60:33247): Connection refused: /127.22.168.60:33247, [2ms] delaying RPC due to: Network error: Failed to connect to peer master-127.22.168.60:33247(127.22.168.60:33247): Connection refused: /127.22.168.60:33247, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: Network error: Failed to connect to peer master-127.22.168.60:33247(127.22.168.60:33247): Connection refused: /127.22.168.60:33247, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: OK, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.60:33247(127.22.168.60:33247): Connection refused: /127.22.168.60:33247, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: Network error: Failed to connect to peer master-127.22.168.60:33247(127.22.168.60:33247): Connection refused: /127.22.168.60:33247, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: OK, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.60:33247(127.22.168.60:33247): Connection refused: /127.22.168.60:33247, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: Network error: Failed to connect to peer master-127.22.168.60:33247(127.22.168.60:33247): Connection refused: /127.22.168.60:33247, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: OK, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.60:33247(127.22.168.60:33247): Connection refused: /127.22.168.60:33247, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: Network error: Failed to connect to peer master-127.22.168.60:33247(127.22.168.60:33247): Connection refused: /127.22.168.60:33247, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: OK, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.60:33247(127.22.168.60:33247): Connection refused: /127.22.168.60:33247, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: Network error: Failed to connect to peer master-127.22.168.60:33247(127.22.168.60:33247): Connection refused: /127.22.168.60:33247, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: OK, [103ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.60:33247(127.22.168.60:33247): Connection refused: /127.22.168.60:33247, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [161ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: Network error: Failed to connect to peer master-127.22.168.60:33247(127.22.168.60:33247): Connection refused: /127.22.168.60:33247, [161ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: OK, [162ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.60:33247(127.22.168.60:33247): Connection refused: /127.22.168.60:33247, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [221ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: Network error: Failed to connect to peer master-127.22.168.60:33247(127.22.168.60:33247): Connection refused: /127.22.168.60:33247, [222ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: OK, [222ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: OK, [222ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.60:33247(127.22.168.60:33247): Connection refused: /127.22.168.60:33247, deferred=Deferred@1658330745(state=PENDING, result=null, callback=org.apache.kudu.client.KuduTransaction$$Lambda$119/48695226@7993d532 -> (continuation of Deferred@13624982 after retry RPC after error@1783099049) -> (continuation of Deferred@1213079118 after retry RPC after error@789801008) -> (continuation of Deferred@1343733778 after retry RPC after error@91246111) -> (continuation of Deferred@1039436945 after retry RPC after error@152183287) -> (continuation of Deferred@1735092119 after retry RPC after error@1617747411) -> (continuation of Deferred@2045068619 after retry RPC after error@1633734169), errback=org.apache.kudu.client.KuduTransaction$$Lambda$120/518121423@5d0b5a7d -> (continuation of Deferred@13624982 after retry RPC after error@1783099049) -> (continuation of Deferred@1213079118 after retry RPC after error@789801008) -> (continuation of Deferred@1343733778 after retry RPC after error@91246111) -> (continuation of Deferred@1039436945 after retry RPC after error@152183287) -> (continuation of Deferred@1735092119 after retry RPC after error@1617747411) -> (continuation of Deferred@2045068619 after retry RPC after error@1633734169)))
21:04:12.767 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=Kudu Master, attempt=8, TimeoutTracker(timeout=250, elapsed=222), Traces: [0ms] sending RPC to server master-127.22.168.60:33247, [2ms] received response from server master-127.22.168.60:33247: Network error: Failed to connect to peer master-127.22.168.60:33247(127.22.168.60:33247): Connection refused: /127.22.168.60:33247, [2ms] delaying RPC due to: Network error: Failed to connect to peer master-127.22.168.60:33247(127.22.168.60:33247): Connection refused: /127.22.168.60:33247, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: Network error: Failed to connect to peer master-127.22.168.60:33247(127.22.168.60:33247): Connection refused: /127.22.168.60:33247, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: OK, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.60:33247(127.22.168.60:33247): Connection refused: /127.22.168.60:33247, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: Network error: Failed to connect to peer master-127.22.168.60:33247(127.22.168.60:33247): Connection refused: /127.22.168.60:33247, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: OK, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.60:33247(127.22.168.60:33247): Connection refused: /127.22.168.60:33247, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: Network error: Failed to connect to peer master-127.22.168.60:33247(127.22.168.60:33247): Connection refused: /127.22.168.60:33247, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: OK, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.60:33247(127.22.168.60:33247): Connection refused: /127.22.168.60:33247, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: Network error: Failed to connect to peer master-127.22.168.60:33247(127.22.168.60:33247): Connection refused: /127.22.168.60:33247, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: OK, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.60:33247(127.22.168.60:33247): Connection refused: /127.22.168.60:33247, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: Network error: Failed to connect to peer master-127.22.168.60:33247(127.22.168.60:33247): Connection refused: /127.22.168.60:33247, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: OK, [103ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.60:33247(127.22.168.60:33247): Connection refused: /127.22.168.60:33247, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [161ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: Network error: Failed to connect to peer master-127.22.168.60:33247(127.22.168.60:33247): Connection refused: /127.22.168.60:33247, [161ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: OK, [162ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.60:33247(127.22.168.60:33247): Connection refused: /127.22.168.60:33247, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [221ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: Network error: Failed to connect to peer master-127.22.168.60:33247(127.22.168.60:33247): Connection refused: /127.22.168.60:33247, [222ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: OK, [222ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: OK, [222ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.60:33247(127.22.168.60:33247): Connection refused: /127.22.168.60:33247, deferred=Deferred@1658330745(state=PENDING, result=null, callback=org.apache.kudu.client.KuduTransaction$$Lambda$119/48695226@7993d532 -> (continuation of Deferred@13624982 after retry RPC after error@1783099049) -> (continuation of Deferred@1213079118 after retry RPC after error@789801008) -> (continuation of Deferred@1343733778 after retry RPC after error@91246111) -> (continuation of Deferred@1039436945 after retry RPC after error@152183287) -> (continuation of Deferred@1735092119 after retry RPC after error@1617747411) -> (continuation of Deferred@2045068619 after retry RPC after error@1633734169), errback=org.apache.kudu.client.KuduTransaction$$Lambda$120/518121423@5d0b5a7d -> (continuation of Deferred@13624982 after retry RPC after error@1783099049) -> (continuation of Deferred@1213079118 after retry RPC after error@789801008) -> (continuation of Deferred@1343733778 after retry RPC after error@91246111) -> (continuation of Deferred@1039436945 after retry RPC after error@152183287) -> (continuation of Deferred@1735092119 after retry RPC after error@1617747411) -> (continuation of Deferred@2045068619 after retry RPC after error@1633734169)))
21:04:12.784 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.60:33247(127.22.168.60:33247)] connecting to peer
21:04:12.785 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.60:33247(127.22.168.60:33247): Connection refused: /127.22.168.60:33247
21:04:12.785 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.60:33247(127.22.168.60:33247)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.60:33247(127.22.168.60:33247): Connection refused: /127.22.168.60:33247
21:04:12.785 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.60:33247: Failed to connect to peer master-127.22.168.60:33247(127.22.168.60:33247): Connection refused: /127.22.168.60:33247
21:04:12.786 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:12.804 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.60:33247(127.22.168.60:33247)] connecting to peer
21:04:12.805 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.60:33247(127.22.168.60:33247): Connection refused: /127.22.168.60:33247
21:04:12.805 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.60:33247(127.22.168.60:33247)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.60:33247(127.22.168.60:33247): Connection refused: /127.22.168.60:33247
21:04:12.806 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.60:33247: Failed to connect to peer master-127.22.168.60:33247(127.22.168.60:33247): Connection refused: /127.22.168.60:33247
21:04:12.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:12.806102 24020 raft_consensus.cc:493] T 00000000000000000000000000000000 P e9dd218ca2244f5a934aac2b7e58eb1d [term 2 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
21:04:12.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:12.806326 24020 raft_consensus.cc:515] T 00000000000000000000000000000000 P e9dd218ca2244f5a934aac2b7e58eb1d [term 2 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e9dd218ca2244f5a934aac2b7e58eb1d" member_type: VOTER last_known_addr { host: "127.22.168.62" port: 37239 } } peers { permanent_uuid: "c520077acf3341e29447c6c2b8d3f74c" member_type: VOTER last_known_addr { host: "127.22.168.61" port: 32947 } } peers { permanent_uuid: "eb664add87f44013b596f4cc01c9a41c" member_type: VOTER last_known_addr { host: "127.22.168.60" port: 33247 } }
21:04:12.807 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:12.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:12.807063 24020 leader_election.cc:290] T 00000000000000000000000000000000 P e9dd218ca2244f5a934aac2b7e58eb1d [CANDIDATE]: Term 3 pre-election: Requested pre-vote from peers c520077acf3341e29447c6c2b8d3f74c (127.22.168.61:32947), eb664add87f44013b596f4cc01c9a41c (127.22.168.60:33247)
21:04:12.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:12.807849 23331 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "e9dd218ca2244f5a934aac2b7e58eb1d" candidate_term: 3 candidate_status { last_received { term: 1 index: 11 } } ignore_live_leader: false dest_uuid: "c520077acf3341e29447c6c2b8d3f74c" is_pre_election: true
21:04:12.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:12.808113 23331 raft_consensus.cc:2468] T 00000000000000000000000000000000 P c520077acf3341e29447c6c2b8d3f74c [term 2 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate e9dd218ca2244f5a934aac2b7e58eb1d in term 2.
21:04:12.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:12.808528 23229 leader_election.cc:304] T 00000000000000000000000000000000 P e9dd218ca2244f5a934aac2b7e58eb1d [CANDIDATE]: Term 3 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: c520077acf3341e29447c6c2b8d3f74c, e9dd218ca2244f5a934aac2b7e58eb1d; no voters:
21:04:12.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:12.808794 24020 raft_consensus.cc:2804] T 00000000000000000000000000000000 P e9dd218ca2244f5a934aac2b7e58eb1d [term 2 FOLLOWER]: Leader pre-election won for term 3
21:04:12.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:12.808905 24020 raft_consensus.cc:493] T 00000000000000000000000000000000 P e9dd218ca2244f5a934aac2b7e58eb1d [term 2 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
21:04:12.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:12.809023 24020 raft_consensus.cc:3060] T 00000000000000000000000000000000 P e9dd218ca2244f5a934aac2b7e58eb1d [term 2 FOLLOWER]: Advancing to term 3
21:04:12.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:12.809336 23229 leader_election.cc:336] T 00000000000000000000000000000000 P e9dd218ca2244f5a934aac2b7e58eb1d [CANDIDATE]: Term 3 pre-election: RPC error from VoteRequest() call to peer eb664add87f44013b596f4cc01c9a41c (127.22.168.60:33247): Network error: Client connection negotiation failed: client connection to 127.22.168.60:33247: connect: Connection refused (error 111)
21:04:12.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:12.811179 24020 raft_consensus.cc:515] T 00000000000000000000000000000000 P e9dd218ca2244f5a934aac2b7e58eb1d [term 3 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e9dd218ca2244f5a934aac2b7e58eb1d" member_type: VOTER last_known_addr { host: "127.22.168.62" port: 37239 } } peers { permanent_uuid: "c520077acf3341e29447c6c2b8d3f74c" member_type: VOTER last_known_addr { host: "127.22.168.61" port: 32947 } } peers { permanent_uuid: "eb664add87f44013b596f4cc01c9a41c" member_type: VOTER last_known_addr { host: "127.22.168.60" port: 33247 } }
21:04:12.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:12.811707 24020 leader_election.cc:290] T 00000000000000000000000000000000 P e9dd218ca2244f5a934aac2b7e58eb1d [CANDIDATE]: Term 3 election: Requested vote from peers c520077acf3341e29447c6c2b8d3f74c (127.22.168.61:32947), eb664add87f44013b596f4cc01c9a41c (127.22.168.60:33247)
21:04:12.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:12.812162 23331 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "e9dd218ca2244f5a934aac2b7e58eb1d" candidate_term: 3 candidate_status { last_received { term: 1 index: 11 } } ignore_live_leader: false dest_uuid: "c520077acf3341e29447c6c2b8d3f74c"
21:04:12.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:12.812386 23331 raft_consensus.cc:3060] T 00000000000000000000000000000000 P c520077acf3341e29447c6c2b8d3f74c [term 2 FOLLOWER]: Advancing to term 3
21:04:12.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:12.813220 23229 leader_election.cc:336] T 00000000000000000000000000000000 P e9dd218ca2244f5a934aac2b7e58eb1d [CANDIDATE]: Term 3 election: RPC error from VoteRequest() call to peer eb664add87f44013b596f4cc01c9a41c (127.22.168.60:33247): Network error: Client connection negotiation failed: client connection to 127.22.168.60:33247: connect: Connection refused (error 111)
21:04:12.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:12.814651 23331 raft_consensus.cc:2468] T 00000000000000000000000000000000 P c520077acf3341e29447c6c2b8d3f74c [term 3 FOLLOWER]: Leader election vote request: Granting yes vote for candidate e9dd218ca2244f5a934aac2b7e58eb1d in term 3.
21:04:12.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:12.815054 23229 leader_election.cc:304] T 00000000000000000000000000000000 P e9dd218ca2244f5a934aac2b7e58eb1d [CANDIDATE]: Term 3 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: c520077acf3341e29447c6c2b8d3f74c, e9dd218ca2244f5a934aac2b7e58eb1d; no voters: eb664add87f44013b596f4cc01c9a41c
21:04:12.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:12.815310 24020 raft_consensus.cc:2804] T 00000000000000000000000000000000 P e9dd218ca2244f5a934aac2b7e58eb1d [term 3 FOLLOWER]: Leader election won for term 3
21:04:12.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:12.815976 24020 raft_consensus.cc:697] T 00000000000000000000000000000000 P e9dd218ca2244f5a934aac2b7e58eb1d [term 3 LEADER]: Becoming Leader. State: Replica: e9dd218ca2244f5a934aac2b7e58eb1d, State: Running, Role: LEADER
21:04:12.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:12.816469 24020 consensus_queue.cc:237] T 00000000000000000000000000000000 P e9dd218ca2244f5a934aac2b7e58eb1d [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 11, Committed index: 11, Last appended: 1.11, Last appended by leader: 11, Current term: 3, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e9dd218ca2244f5a934aac2b7e58eb1d" member_type: VOTER last_known_addr { host: "127.22.168.62" port: 37239 } } peers { permanent_uuid: "c520077acf3341e29447c6c2b8d3f74c" member_type: VOTER last_known_addr { host: "127.22.168.61" port: 32947 } } peers { permanent_uuid: "eb664add87f44013b596f4cc01c9a41c" member_type: VOTER last_known_addr { host: "127.22.168.60" port: 33247 } }
21:04:12.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:12.817813 24031 sys_catalog.cc:455] T 00000000000000000000000000000000 P e9dd218ca2244f5a934aac2b7e58eb1d [sys.catalog]: SysCatalogTable state changed. Reason: New leader e9dd218ca2244f5a934aac2b7e58eb1d. Latest consensus state: current_term: 3 leader_uuid: "e9dd218ca2244f5a934aac2b7e58eb1d" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e9dd218ca2244f5a934aac2b7e58eb1d" member_type: VOTER last_known_addr { host: "127.22.168.62" port: 37239 } } peers { permanent_uuid: "c520077acf3341e29447c6c2b8d3f74c" member_type: VOTER last_known_addr { host: "127.22.168.61" port: 32947 } } peers { permanent_uuid: "eb664add87f44013b596f4cc01c9a41c" member_type: VOTER last_known_addr { host: "127.22.168.60" port: 33247 } } }
21:04:12.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:12.818032 24031 sys_catalog.cc:458] T 00000000000000000000000000000000 P e9dd218ca2244f5a934aac2b7e58eb1d [sys.catalog]: This master's current role is: LEADER
21:04:12.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:12.818811 24032 catalog_manager.cc:1485] Loading table and tablet metadata into memory...
21:04:12.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:12.822031 24032 catalog_manager.cc:679] Loaded metadata for table txn_manager_fallback_rolling [id=03fcdef52a6f4a7099b05a3d2fa91c76]
21:04:12.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:12.822660 24032 catalog_manager.cc:679] Loaded metadata for table kudu_system.kudu_transactions [id=c54eb72594a5488d8dd9c9882da41ff4]
21:04:12.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:12.825531 24032 tablet_loader.cc:96] loaded metadata for tablet 01aa0e192c434b468ec7ed90d48563b3 (table txn_manager_fallback_rolling [id=03fcdef52a6f4a7099b05a3d2fa91c76])
21:04:12.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:12.826047 24032 tablet_loader.cc:96] loaded metadata for tablet 260c684b258041bab84c3ff36578f96b (table txn_manager_fallback_rolling [id=03fcdef52a6f4a7099b05a3d2fa91c76])
21:04:12.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:12.826431 24032 tablet_loader.cc:96] loaded metadata for tablet 581dbe2f34ca4a7981755fa5f4b6b6ca (table kudu_system.kudu_transactions [id=c54eb72594a5488d8dd9c9882da41ff4])
21:04:12.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:12.826725 24032 catalog_manager.cc:1494] Initializing Kudu cluster ID...
21:04:12.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:12.827932 24032 catalog_manager.cc:1269] Loaded cluster ID: 43f2ae537b96433b8f15997b1c5c1cd1
21:04:12.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:12.828040 24032 catalog_manager.cc:1505] Initializing Kudu internal certificate authority...
21:04:12.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:12.829620 24032 catalog_manager.cc:1514] Loading token signing keys...
21:04:12.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:12.830880 24032 catalog_manager.cc:6033] T 00000000000000000000000000000000 P e9dd218ca2244f5a934aac2b7e58eb1d: Loaded TSK: 0
21:04:12.831 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.60:33247(127.22.168.60:33247)] connecting to peer
21:04:12.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:12.832177 24032 catalog_manager.cc:1524] Initializing in-progress tserver states...
21:04:12.832 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.60:33247(127.22.168.60:33247): Connection refused: /127.22.168.60:33247
21:04:12.832 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.60:33247(127.22.168.60:33247)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.60:33247(127.22.168.60:33247): Connection refused: /127.22.168.60:33247
21:04:12.832 [INFO - pool-40-thread-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.60:33247: Failed to connect to peer master-127.22.168.60:33247(127.22.168.60:33247): Connection refused: /127.22.168.60:33247
21:04:12.834 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:12.845 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.60:33247(127.22.168.60:33247)] connecting to peer
21:04:12.846 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.60:33247(127.22.168.60:33247): Connection refused: /127.22.168.60:33247
21:04:12.846 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.60:33247(127.22.168.60:33247)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.60:33247(127.22.168.60:33247): Connection refused: /127.22.168.60:33247
21:04:12.846 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.60:33247: Failed to connect to peer master-127.22.168.60:33247(127.22.168.60:33247): Connection refused: /127.22.168.60:33247
21:04:12.848 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet Kudu Master for table 'Kudu Master' with partition [<start>, <end>)
21:04:12.849 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=4, TimeoutTracker(timeout=250, elapsed=64), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: Network error: Failed to connect to peer master-127.22.168.60:33247(127.22.168.60:33247): Connection refused: /127.22.168.60:33247, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: OK, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.60:33247(127.22.168.60:33247): Connection refused: /127.22.168.60:33247, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: Network error: Failed to connect to peer master-127.22.168.60:33247(127.22.168.60:33247): Connection refused: /127.22.168.60:33247, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: OK, [23ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.60:33247(127.22.168.60:33247): Connection refused: /127.22.168.60:33247, [45ms] refreshing cache from master, [45ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [47ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [47ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: OK, [48ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: Network error: Failed to connect to peer master-127.22.168.60:33247(127.22.168.60:33247): Connection refused: /127.22.168.60:33247, [48ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [50ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: OK, [50ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.60:33247(127.22.168.60:33247): Connection refused: /127.22.168.60:33247, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: Network error: Failed to connect to peer master-127.22.168.60:33247(127.22.168.60:33247): Connection refused: /127.22.168.60:33247, [63ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: OK, [63ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: OK, deferred=Deferred@1335974344(state=PENDING, result=null, callback=(continuation of Deferred@1574641526 after retry RPC after error@271678325) -> (continuation of Deferred@566333191 after retry RPC after error@1693187566) -> (continuation of Deferred@1632534186 after retry RPC after error@2050972453), errback=(continuation of Deferred@1574641526 after retry RPC after error@271678325) -> (continuation of Deferred@566333191 after retry RPC after error@1693187566) -> (continuation of Deferred@1632534186 after retry RPC after error@2050972453))) after lookup
21:04:12.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:12.886507 23241 catalog_manager.cc:2257] Servicing CreateTable request from {username='slave'} at 127.0.0.1:50576:
21:04:12.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "kudu_system.kudu_transactions"
21:04:12.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) schema {
21:04:12.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
21:04:12.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_id"
21:04:12.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT64
21:04:12.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
21:04:12.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
21:04:12.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
21:04:12.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
21:04:12.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
21:04:12.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
21:04:12.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
21:04:12.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
21:04:12.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "entry_type"
21:04:12.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT8
21:04:12.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
21:04:12.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
21:04:12.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
21:04:12.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
21:04:12.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
21:04:12.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
21:04:12.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
21:04:12.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
21:04:12.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "identifier"
21:04:12.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
21:04:12.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
21:04:12.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
21:04:12.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
21:04:12.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
21:04:12.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
21:04:12.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
21:04:12.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
21:04:12.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
21:04:12.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "metadata"
21:04:12.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
21:04:12.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
21:04:12.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
21:04:12.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
21:04:12.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
21:04:12.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
21:04:12.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
21:04:12.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
21:04:12.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
21:04:12.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) num_replicas: 3
21:04:12.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) split_rows_range_bounds {
21:04:12.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) rows: "<redacted>""\006\001\000\000\000\000\000\000\000\000\007\001@B\017\000\000\000\000\000"
21:04:12.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) indirect_data: "<redacted>"""
21:04:12.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
21:04:12.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) partition_schema {
21:04:12.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) range_schema {
21:04:12.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
21:04:12.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_id"
21:04:12.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
21:04:12.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
21:04:12.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
21:04:12.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) table_type: TXN_STATUS_TABLE
21:04:12.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:12.888808 23241 catalog_manager.cc:7011] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table kudu_system.kudu_transactions in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
21:04:12.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:12.939304 23331 raft_consensus.cc:1275] T 00000000000000000000000000000000 P c520077acf3341e29447c6c2b8d3f74c [term 3 FOLLOWER]: Refusing update from remote peer e9dd218ca2244f5a934aac2b7e58eb1d: Log matching property violated. Preceding OpId in replica: term: 1 index: 11. Preceding OpId from leader: term: 3 index: 12. (index mismatch)
21:04:12.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:12.940162 24020 consensus_queue.cc:1048] T 00000000000000000000000000000000 P e9dd218ca2244f5a934aac2b7e58eb1d [LEADER]: Connected to new peer: Peer: permanent_uuid: "c520077acf3341e29447c6c2b8d3f74c" member_type: VOTER last_known_addr { host: "127.22.168.61" port: 32947 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 12, Last known committed idx: 11, Time since last communication: 0.000s
21:04:12.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:12.941461 23229 consensus_peers.cc:597] T 00000000000000000000000000000000 P e9dd218ca2244f5a934aac2b7e58eb1d -> Peer eb664add87f44013b596f4cc01c9a41c (127.22.168.60:33247): Couldn't send request to peer eb664add87f44013b596f4cc01c9a41c. Status: Network error: Client connection negotiation failed: client connection to 127.22.168.60:33247: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
21:04:12.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:12.941804 24019 sys_catalog.cc:455] T 00000000000000000000000000000000 P c520077acf3341e29447c6c2b8d3f74c [sys.catalog]: SysCatalogTable state changed. Reason: New leader e9dd218ca2244f5a934aac2b7e58eb1d. Latest consensus state: current_term: 3 leader_uuid: "e9dd218ca2244f5a934aac2b7e58eb1d" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e9dd218ca2244f5a934aac2b7e58eb1d" member_type: VOTER last_known_addr { host: "127.22.168.62" port: 37239 } } peers { permanent_uuid: "c520077acf3341e29447c6c2b8d3f74c" member_type: VOTER last_known_addr { host: "127.22.168.61" port: 32947 } } peers { permanent_uuid: "eb664add87f44013b596f4cc01c9a41c" member_type: VOTER last_known_addr { host: "127.22.168.60" port: 33247 } } }
21:04:12.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:12.942085 24019 sys_catalog.cc:458] T 00000000000000000000000000000000 P c520077acf3341e29447c6c2b8d3f74c [sys.catalog]: This master's current role is: FOLLOWER
21:04:12.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:12.943374 24020 sys_catalog.cc:455] T 00000000000000000000000000000000 P e9dd218ca2244f5a934aac2b7e58eb1d [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 3 leader_uuid: "e9dd218ca2244f5a934aac2b7e58eb1d" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e9dd218ca2244f5a934aac2b7e58eb1d" member_type: VOTER last_known_addr { host: "127.22.168.62" port: 37239 } } peers { permanent_uuid: "c520077acf3341e29447c6c2b8d3f74c" member_type: VOTER last_known_addr { host: "127.22.168.61" port: 32947 } } peers { permanent_uuid: "eb664add87f44013b596f4cc01c9a41c" member_type: VOTER last_known_addr { host: "127.22.168.60" port: 33247 } } }
21:04:12.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:12.943581 24020 sys_catalog.cc:458] T 00000000000000000000000000000000 P e9dd218ca2244f5a934aac2b7e58eb1d [sys.catalog]: This master's current role is: LEADER
21:04:12.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:12.943993 24019 sys_catalog.cc:455] T 00000000000000000000000000000000 P c520077acf3341e29447c6c2b8d3f74c [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 3 leader_uuid: "e9dd218ca2244f5a934aac2b7e58eb1d" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e9dd218ca2244f5a934aac2b7e58eb1d" member_type: VOTER last_known_addr { host: "127.22.168.62" port: 37239 } } peers { permanent_uuid: "c520077acf3341e29447c6c2b8d3f74c" member_type: VOTER last_known_addr { host: "127.22.168.61" port: 32947 } } peers { permanent_uuid: "eb664add87f44013b596f4cc01c9a41c" member_type: VOTER last_known_addr { host: "127.22.168.60" port: 33247 } } }
21:04:12.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:12.944182 24019 sys_catalog.cc:458] T 00000000000000000000000000000000 P c520077acf3341e29447c6c2b8d3f74c [sys.catalog]: This master's current role is: FOLLOWER
21:04:13.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:13.477643 23229 consensus_peers.cc:597] T 00000000000000000000000000000000 P e9dd218ca2244f5a934aac2b7e58eb1d -> Peer eb664add87f44013b596f4cc01c9a41c (127.22.168.60:33247): Couldn't send request to peer eb664add87f44013b596f4cc01c9a41c. Status: Network error: Client connection negotiation failed: client connection to 127.22.168.60:33247: connect: Connection refused (error 111). This is attempt 6: this message will repeat every 5th retry.
21:04:13.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:13.671746 23229 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.22.168.60:33247: connect: Connection refused (error 111) [suppressed 18 similar messages]
21:04:13.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:13.957319 23229 consensus_peers.cc:597] T 00000000000000000000000000000000 P e9dd218ca2244f5a934aac2b7e58eb1d -> Peer eb664add87f44013b596f4cc01c9a41c (127.22.168.60:33247): Couldn't send request to peer eb664add87f44013b596f4cc01c9a41c. Status: Network error: Client connection negotiation failed: client connection to 127.22.168.60:33247: connect: Connection refused (error 111). This is attempt 11: this message will repeat every 5th retry.
21:04:14.214 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 01aa0e192c434b468ec7ed90d48563b3 for table 'txn_manager_fallback_rolling' with partition [0x00000000, 0x00000001)
21:04:14.214 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 260c684b258041bab84c3ff36578f96b for table 'txn_manager_fallback_rolling' with partition [0x00000001, 0x00000002)
21:04:14.214 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x00000000, upperBoundPartitionKey=0x00000001, ttl=299999, tablet=01aa0e192c434b468ec7ed90d48563b3@[1acec06b763242d082ddb75231fc4eca(127.22.168.2:37409),210cdae6f0e44d83acf5641ce563b738(127.22.168.1:42241),e24858bd4ab14759b2252ddad7f0e5cd(127.22.168.3:35191)[L]]}, Tablet{lowerBoundPartitionKey=0x00000001, upperBoundPartitionKey=0x00000002, ttl=299999, tablet=260c684b258041bab84c3ff36578f96b@[1acec06b763242d082ddb75231fc4eca(127.22.168.2:37409)[L],210cdae6f0e44d83acf5641ce563b738(127.22.168.1:42241),e24858bd4ab14759b2252ddad7f0e5cd(127.22.168.3:35191)]}, NonCoveredRange{lowerBoundPartitionKey=0x00000002, upperBoundPartitionKey=0x, ttl=299999}]
21:04:14.214 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=null, attempt=1, sequence_id=-1, TimeoutTracker(timeout=30000, elapsed=2), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.168.62:37239, [1ms] Sub RPC GetTableLocations: received response from server master-127.22.168.62:37239: OK, deferred=null) after lookup
21:04:14.214 [DEBUG - kudu-nio-0] (Connection.java:748) [peer e24858bd4ab14759b2252ddad7f0e5cd(127.22.168.3:35191)] connecting to peer
21:04:14.215 [DEBUG - kudu-nio-1] (Connection.java:757) [peer e24858bd4ab14759b2252ddad7f0e5cd(127.22.168.3:35191)] Successfully connected to peer
21:04:14.226 [DEBUG - kudu-nio-1] (Negotiator.java:919) Authenticated connection [id: 0x20ad6a2a, L:/127.0.0.1:56866 - R:/127.22.168.3:35191] using TOKEN/null
21:04:14.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:14.231030 23858 master_proxy_rpc.cc:203] Re-attempting GetTableSchema request to leader Master (127.22.168.60:33247)
21:04:14.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:14.264314 23654 heartbeater.cc:499] Master 127.22.168.62:37239 was elected leader, sending a full tablet report...
21:04:14.271 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.22.168.60:33247
21:04:14.272 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 2
}
}
21:04:14.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:14.272990 23200 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskRxIrUU/build/asan/bin/kudu
21:04:14.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskRxIrUU/build/asan/bin/kudu
21:04:14.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskRxIrUU/test-tmp/mini-kudu-cluster8143392791847762722/master-2/wal
21:04:14.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskRxIrUU/test-tmp/mini-kudu-cluster8143392791847762722/master-2/data
21:04:14.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
21:04:14.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
21:04:14.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
21:04:14.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:04:14.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:04:14.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:04:14.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskRxIrUU/test-tmp/mini-kudu-cluster8143392791847762722/master-2/logs
21:04:14.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskRxIrUU/test-tmp/mini-kudu-cluster8143392791847762722/master-2/data/info.pb
21:04:14.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:04:14.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
21:04:14.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
21:04:14.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
21:04:14.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
21:04:14.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:04:14.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:04:14.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:04:14.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
21:04:14.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
21:04:14.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
21:04:14.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
21:04:14.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.22.168.60:33247
21:04:14.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.22.168.60
21:04:14.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=33853
21:04:14.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.22.168.20:37293
21:04:14.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:04:14.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:04:14.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:04:14.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
21:04:14.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.22.168.62:37239,127.22.168.61:32947,127.22.168.60:33247
21:04:14.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
21:04:14.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:43617/jwks.json
21:04:14.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled
21:04:14.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100 with env {}
21:04:14.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:14.489746 23229 consensus_peers.cc:597] T 00000000000000000000000000000000 P e9dd218ca2244f5a934aac2b7e58eb1d -> Peer eb664add87f44013b596f4cc01c9a41c (127.22.168.60:33247): Couldn't send request to peer eb664add87f44013b596f4cc01c9a41c. Status: Network error: Client connection negotiation failed: client connection to 127.22.168.60:33247: connect: Connection refused (error 111). This is attempt 16: this message will repeat every 5th retry.
21:04:14.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:14.605221 23211 master_runner.cc:186] Error getting master registration for 127.22.168.60:33247: Network error: Client connection negotiation failed: client connection to 127.22.168.60:33247: connect: Connection refused (error 111),
21:04:14.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:14.605445 23211 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
21:04:14.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:14.614246 24071 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
21:04:14.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:14.614599 24071 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
21:04:14.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:14.614717 24071 flags.cc:425] Enabled unsafe flag: --never_fsync=true
21:04:14.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:14.626691 24071 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
21:04:14.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:14.626803 24071 flags.cc:425] Enabled experimental flag: --txn_manager_enabled=true
21:04:14.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:14.626868 24071 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
21:04:14.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:14.626920 24071 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
21:04:14.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:14.627008 24071 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
21:04:14.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:14.627066 24071 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
21:04:14.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:14.627132 24071 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:43617/jwks.json
21:04:14.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:14.638170 24071 master_runner.cc:386] Master server non-default flags:
21:04:14.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:04:14.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.22.168.20:37293
21:04:14.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:04:14.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:04:14.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100
21:04:14.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskRxIrUU/test-tmp/mini-kudu-cluster8143392791847762722/master-2/data
21:04:14.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskRxIrUU/test-tmp/mini-kudu-cluster8143392791847762722/master-2/wal
21:04:14.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
21:04:14.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.22.168.62:37239,127.22.168.61:32947,127.22.168.60:33247
21:04:14.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled=true
21:04:14.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:04:14.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:04:14.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
21:04:14.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.22.168.60:33247
21:04:14.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
21:04:14.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
21:04:14.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
21:04:14.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:43617/jwks.json
21:04:14.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:04:14.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:04:14.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskRxIrUU/test-tmp/mini-kudu-cluster8143392791847762722/master-2/data/info.pb
21:04:14.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.22.168.60
21:04:14.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=33853
21:04:14.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
21:04:14.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:04:14.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
21:04:14.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
21:04:14.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:04:14.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskRxIrUU/test-tmp/mini-kudu-cluster8143392791847762722/master-2/logs
21:04:14.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:04:14.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
21:04:14.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
21:04:14.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
21:04:14.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
21:04:14.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 60f5e5267b92c39485a66121d3ce3cc7ef57b0e0
21:04:14.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
21:04:14.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 28 Oct 2025 20:43:24 UTC on 24a791456cd2
21:04:14.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 8764
21:04:14.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
21:04:14.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:14.639829 24071 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
21:04:14.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:14.641678 24071 file_cache.cc:492] Constructed file cache file cache with capacity 419430
21:04:14.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:14.650079 24076 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:04:14.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:14.650084 24077 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:04:14.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:14.650267 24079 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:04:14.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:14.651122 24071 server_base.cc:1047] running on GCE node
21:04:14.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:14.651882 24071 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
21:04:14.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:14.653398 24071 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
21:04:14.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:14.654613 24071 hybrid_clock.cc:648] HybridClock initialized: now 1761685454654555 us; error 37 us; skew 500 ppm
21:04:14.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:14.654943 24071 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
21:04:14.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:14.657182 24071 webserver.cc:492] Webserver started at http://127.22.168.60:33853/ using document root <none> and password file <none>
21:04:14.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:14.657862 24071 fs_manager.cc:362] Metadata directory not provided
21:04:14.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:14.657995 24071 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
21:04:14.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:14.663285 24071 fs_manager.cc:714] Time spent opening directory manager: real 0.003s user 0.005s sys 0.000s
21:04:14.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:14.666250 24085 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
21:04:14.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:14.667578 24071 fs_manager.cc:730] Time spent opening block manager: real 0.003s user 0.001s sys 0.000s
21:04:14.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:14.667735 24071 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskRxIrUU/test-tmp/mini-kudu-cluster8143392791847762722/master-2/data,/tmp/dist-test-taskRxIrUU/test-tmp/mini-kudu-cluster8143392791847762722/master-2/wal
21:04:14.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "eb664add87f44013b596f4cc01c9a41c"
21:04:14.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-28 21:04:09 on dist-test-slave-vx1b"
21:04:14.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:14.668556 24071 fs_report.cc:389] FS layout report
21:04:14.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
21:04:14.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskRxIrUU/test-tmp/mini-kudu-cluster8143392791847762722/master-2/wal
21:04:14.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskRxIrUU/test-tmp/mini-kudu-cluster8143392791847762722/master-2/wal
21:04:14.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskRxIrUU/test-tmp/mini-kudu-cluster8143392791847762722/master-2/data/data
21:04:14.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
21:04:14.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
21:04:14.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
21:04:14.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
21:04:14.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
21:04:14.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
21:04:14.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
21:04:14.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
21:04:14.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
21:04:14.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
21:04:14.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
21:04:14.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:14.691812 24071 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
21:04:14.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:14.692728 24071 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
21:04:14.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:14.693017 24071 kserver.cc:163] Server-wide thread pool size limit: 3276
21:04:14.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:14.701211 23795 heartbeater.cc:499] Master 127.22.168.62:37239 was elected leader, sending a full tablet report...
21:04:14.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:14.720232 24071 rpc_server.cc:307] RPC server started. Bound to: 127.22.168.60:33247
21:04:14.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:14.720265 24147 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.22.168.60:33247 every 8 connection(s)
21:04:14.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:14.721786 24071 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskRxIrUU/test-tmp/mini-kudu-cluster8143392791847762722/master-2/data/info.pb
21:04:14.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:14.726500 23200 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskRxIrUU/build/asan/bin/kudu as pid 24071
21:04:14.726 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
21:04:14.727 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.22.168.62:37239
21:04:14.727 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 0
}
}
21:04:14.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:14.727348 23200 external_mini_cluster.cc:1699] Attempting to check leaks for /tmp/dist-test-taskRxIrUU/build/asan/bin/kudu pid 23211
21:04:14.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:14.728547 23282 generic_service.cc:196] Checking for leaks (request via RPC)
21:04:14.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:14.729849 24148 sys_catalog.cc:263] Verifying existing consensus state
21:04:14.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:14.733912 24148 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P eb664add87f44013b596f4cc01c9a41c: Bootstrap starting.
21:04:14.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:14.749975 24148 log.cc:826] T 00000000000000000000000000000000 P eb664add87f44013b596f4cc01c9a41c: Log is configured to *not* fsync() on all Append() calls
21:04:14.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:14.763209 24148 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P eb664add87f44013b596f4cc01c9a41c: Bootstrap replayed 1/1 log segments. Stats: ops{read=11 overwritten=0 applied=11 ignored=0} inserts{seen=8 ignored=0} mutations{seen=6 ignored=0} orphaned_commits=0. Pending: 0 replicates
21:04:14.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:14.763703 24148 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P eb664add87f44013b596f4cc01c9a41c: Bootstrap complete.
21:04:14.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:14.771104 24148 raft_consensus.cc:359] T 00000000000000000000000000000000 P eb664add87f44013b596f4cc01c9a41c [term 1 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e9dd218ca2244f5a934aac2b7e58eb1d" member_type: VOTER last_known_addr { host: "127.22.168.62" port: 37239 } } peers { permanent_uuid: "c520077acf3341e29447c6c2b8d3f74c" member_type: VOTER last_known_addr { host: "127.22.168.61" port: 32947 } } peers { permanent_uuid: "eb664add87f44013b596f4cc01c9a41c" member_type: VOTER last_known_addr { host: "127.22.168.60" port: 33247 } }
21:04:14.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:14.771770 24148 raft_consensus.cc:740] T 00000000000000000000000000000000 P eb664add87f44013b596f4cc01c9a41c [term 1 FOLLOWER]: Becoming Follower/Learner. State: Replica: eb664add87f44013b596f4cc01c9a41c, State: Initialized, Role: FOLLOWER
21:04:14.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:14.772401 24148 consensus_queue.cc:260] T 00000000000000000000000000000000 P eb664add87f44013b596f4cc01c9a41c [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 11, Last appended: 1.11, Last appended by leader: 11, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e9dd218ca2244f5a934aac2b7e58eb1d" member_type: VOTER last_known_addr { host: "127.22.168.62" port: 37239 } } peers { permanent_uuid: "c520077acf3341e29447c6c2b8d3f74c" member_type: VOTER last_known_addr { host: "127.22.168.61" port: 32947 } } peers { permanent_uuid: "eb664add87f44013b596f4cc01c9a41c" member_type: VOTER last_known_addr { host: "127.22.168.60" port: 33247 } }
21:04:14.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:14.773228 24152 sys_catalog.cc:455] T 00000000000000000000000000000000 P eb664add87f44013b596f4cc01c9a41c [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 1 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e9dd218ca2244f5a934aac2b7e58eb1d" member_type: VOTER last_known_addr { host: "127.22.168.62" port: 37239 } } peers { permanent_uuid: "c520077acf3341e29447c6c2b8d3f74c" member_type: VOTER last_known_addr { host: "127.22.168.61" port: 32947 } } peers { permanent_uuid: "eb664add87f44013b596f4cc01c9a41c" member_type: VOTER last_known_addr { host: "127.22.168.60" port: 33247 } } }
21:04:14.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:14.773530 24152 sys_catalog.cc:458] T 00000000000000000000000000000000 P eb664add87f44013b596f4cc01c9a41c [sys.catalog]: This master's current role is: FOLLOWER
21:04:14.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:14.774479 24148 sys_catalog.cc:565] T 00000000000000000000000000000000 P eb664add87f44013b596f4cc01c9a41c [sys.catalog]: configured and running, proceeding with master startup.
21:04:14.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:14.775307 23937 heartbeater.cc:499] Master 127.22.168.62:37239 was elected leader, sending a full tablet report...
21:04:14.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:14.793709 24165 catalog_manager.cc:1269] Loaded cluster ID: 43f2ae537b96433b8f15997b1c5c1cd1
21:04:14.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:14.793820 24165 catalog_manager.cc:1562] T 00000000000000000000000000000000 P eb664add87f44013b596f4cc01c9a41c: loading cluster ID for follower catalog manager: success
21:04:14.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:14.794396 23936 heartbeater.cc:344] Connected to a master server at 127.22.168.60:33247
21:04:14.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:14.798065 24103 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "e24858bd4ab14759b2252ddad7f0e5cd" instance_seqno: 1761685451297765) as {username='slave'} at 127.22.168.3:58155; Asking this server to re-register.
21:04:14.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:14.798689 24165 catalog_manager.cc:1584] T 00000000000000000000000000000000 P eb664add87f44013b596f4cc01c9a41c: acquiring CA information for follower catalog manager: success
21:04:14.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:14.799191 23936 heartbeater.cc:461] Registering TS with master...
21:04:14.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:14.801100 24103 ts_manager.cc:194] Registered new tserver with Master: e24858bd4ab14759b2252ddad7f0e5cd (127.22.168.3:35191)
21:04:14.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:14.802279 24165 catalog_manager.cc:1612] T 00000000000000000000000000000000 P eb664add87f44013b596f4cc01c9a41c: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
21:04:14.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:14.867540 23200 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskRxIrUU/build/asan/bin/kudu with pid 23211
21:04:14.900 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.22.168.62:37239(127.22.168.62:37239)] handling channelInactive
21:04:14.900 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state READY due to: connection closed
21:04:14.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:14.901129 24154 negotiation.cc:337] Failed RPC negotiation. Trace:
21:04:14.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1028 21:04:14.865360 (+ 0us) reactor.cc:625] Submitting negotiation task for server connection from 127.0.0.1:49280 (local address 127.22.168.60:33247)
21:04:14.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1028 21:04:14.865554 (+ 194us) server_negotiation.cc:207] Beginning negotiation
21:04:14.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1028 21:04:14.865560 (+ 6us) server_negotiation.cc:400] Waiting for connection header
21:04:14.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1028 21:04:14.865581 (+ 21us) server_negotiation.cc:408] Connection header received
21:04:14.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1028 21:04:14.865652 (+ 71us) server_negotiation.cc:366] Received NEGOTIATE NegotiatePB request
21:04:14.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1028 21:04:14.865659 (+ 7us) server_negotiation.cc:462] Received NEGOTIATE request from client
21:04:14.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1028 21:04:14.865764 (+ 105us) server_negotiation.cc:378] Sending NEGOTIATE NegotiatePB response
21:04:14.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1028 21:04:14.865908 (+ 144us) server_negotiation.cc:227] Negotiated authn=CERTIFICATE
21:04:14.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1028 21:04:14.868944 (+ 3036us) server_negotiation.cc:366] Received TLS_HANDSHAKE NegotiatePB request
21:04:14.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1028 21:04:14.870409 (+ 1465us) server_negotiation.cc:378] Sending TLS_HANDSHAKE NegotiatePB response
21:04:14.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1028 21:04:14.899622 (+ 29213us) negotiation.cc:327] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.0.0.1:49280: BlockingRecv error: recv error from unknown peer: Transport endpoint is not connected (error 107)
21:04:14.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":87}
21:04:14.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:14.901957 24166 negotiation.cc:337] Failed RPC negotiation. Trace:
21:04:14.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1028 21:04:14.786546 (+ 0us) reactor.cc:625] Submitting negotiation task for client connection to 127.22.168.62:37239 (local address 127.0.0.1:50582)
21:04:14.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1028 21:04:14.787314 (+ 768us) negotiation.cc:107] Waiting for socket to connect
21:04:14.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1028 21:04:14.787351 (+ 37us) client_negotiation.cc:174] Beginning negotiation
21:04:14.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1028 21:04:14.787484 (+ 133us) client_negotiation.cc:252] Sending NEGOTIATE NegotiatePB request
21:04:14.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1028 21:04:14.899621 (+112137us) negotiation.cc:327] Negotiation complete: Network error: Client connection negotiation failed: client connection to 127.22.168.62:37239: BlockingRecv error: recv error from unknown peer: Transport endpoint is not connected (error 107)
21:04:14.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"client-negotiator.queue_time_us":554,"thread_start_us":389,"threads_started":1}
21:04:14.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:14.902869 23308 connection.cc:537] server connection from 127.0.0.1:36974 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
21:04:14.905 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
21:04:14.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:14.903602 24071 master_runner.cc:186] Error getting master registration for 127.22.168.62:37239: Network error: Client connection negotiation failed: client connection to 127.22.168.62:37239: BlockingRecv error: recv error from unknown peer: Transport endpoint is not connected (error 107),
21:04:14.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:14.909545 24071 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
21:04:14.984 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:14.985 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:14.985 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:14.985 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location master-127.22.168.62:37239(127.22.168.62:37239) for tablet Kudu Master: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:15.004 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:15.004 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.60:33247(127.22.168.60:33247)] connecting to peer
21:04:15.005 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:15.005 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:15.005 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:15.005 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.22.168.60:33247(127.22.168.60:33247)] Successfully connected to peer
21:04:15.022 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0x4cf198b6, L:/127.0.0.1:49286 - R:/127.22.168.60:33247] using TOKEN/null
21:04:15.023 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:15.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:15.034462 24152 raft_consensus.cc:493] T 00000000000000000000000000000000 P eb664add87f44013b596f4cc01c9a41c [term 1 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
21:04:15.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:15.034812 24152 raft_consensus.cc:515] T 00000000000000000000000000000000 P eb664add87f44013b596f4cc01c9a41c [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e9dd218ca2244f5a934aac2b7e58eb1d" member_type: VOTER last_known_addr { host: "127.22.168.62" port: 37239 } } peers { permanent_uuid: "c520077acf3341e29447c6c2b8d3f74c" member_type: VOTER last_known_addr { host: "127.22.168.61" port: 32947 } } peers { permanent_uuid: "eb664add87f44013b596f4cc01c9a41c" member_type: VOTER last_known_addr { host: "127.22.168.60" port: 33247 } }
21:04:15.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:15.037125 24152 leader_election.cc:290] T 00000000000000000000000000000000 P eb664add87f44013b596f4cc01c9a41c [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers e9dd218ca2244f5a934aac2b7e58eb1d (127.22.168.62:37239), c520077acf3341e29447c6c2b8d3f74c (127.22.168.61:32947)
21:04:15.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:15.038503 24086 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.22.168.62:37239: connect: Connection refused (error 111)
21:04:15.044 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:15.046 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:15.046 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:15.046 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:15.050 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:15.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:15.056284 24086 leader_election.cc:336] T 00000000000000000000000000000000 P eb664add87f44013b596f4cc01c9a41c [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer e9dd218ca2244f5a934aac2b7e58eb1d (127.22.168.62:37239): Network error: Client connection negotiation failed: client connection to 127.22.168.62:37239: connect: Connection refused (error 111)
21:04:15.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:15.061668 23331 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "eb664add87f44013b596f4cc01c9a41c" candidate_term: 2 candidate_status { last_received { term: 1 index: 11 } } ignore_live_leader: false dest_uuid: "c520077acf3341e29447c6c2b8d3f74c" is_pre_election: true
21:04:15.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:15.062709 24089 leader_election.cc:304] T 00000000000000000000000000000000 P eb664add87f44013b596f4cc01c9a41c [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: eb664add87f44013b596f4cc01c9a41c; no voters: c520077acf3341e29447c6c2b8d3f74c, e9dd218ca2244f5a934aac2b7e58eb1d
21:04:15.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:15.063165 24152 raft_consensus.cc:2749] T 00000000000000000000000000000000 P eb664add87f44013b596f4cc01c9a41c [term 1 FOLLOWER]: Leader pre-election lost for term 2. Reason: could not achieve majority
21:04:15.064 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:15.067 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:15.067 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:15.067 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:15.067 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:15.084 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:15.085 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:15.085 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:15.086 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:15.087 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:15.104 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:15.107 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:15.107 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:15.107 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:15.107 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:15.145 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:15.146 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:15.146 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:15.146 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:15.148 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:15.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:15.168948 24172 raft_consensus.cc:493] T 00000000000000000000000000000000 P c520077acf3341e29447c6c2b8d3f74c [term 3 FOLLOWER]: Starting pre-election (detected failure of leader e9dd218ca2244f5a934aac2b7e58eb1d)
21:04:15.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:15.169199 24172 raft_consensus.cc:515] T 00000000000000000000000000000000 P c520077acf3341e29447c6c2b8d3f74c [term 3 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e9dd218ca2244f5a934aac2b7e58eb1d" member_type: VOTER last_known_addr { host: "127.22.168.62" port: 37239 } } peers { permanent_uuid: "c520077acf3341e29447c6c2b8d3f74c" member_type: VOTER last_known_addr { host: "127.22.168.61" port: 32947 } } peers { permanent_uuid: "eb664add87f44013b596f4cc01c9a41c" member_type: VOTER last_known_addr { host: "127.22.168.60" port: 33247 } }
21:04:15.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:15.170019 24172 leader_election.cc:290] T 00000000000000000000000000000000 P c520077acf3341e29447c6c2b8d3f74c [CANDIDATE]: Term 4 pre-election: Requested pre-vote from peers e9dd218ca2244f5a934aac2b7e58eb1d (127.22.168.62:37239), eb664add87f44013b596f4cc01c9a41c (127.22.168.60:33247)
21:04:15.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:15.171015 23307 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.22.168.62:37239: connect: Connection refused (error 111) [suppressed 6 similar messages]
21:04:15.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:15.173024 23307 leader_election.cc:336] T 00000000000000000000000000000000 P c520077acf3341e29447c6c2b8d3f74c [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer e9dd218ca2244f5a934aac2b7e58eb1d (127.22.168.62:37239): Network error: Client connection negotiation failed: client connection to 127.22.168.62:37239: connect: Connection refused (error 111)
21:04:15.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:15.177883 24113 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "c520077acf3341e29447c6c2b8d3f74c" candidate_term: 4 candidate_status { last_received { term: 3 index: 12 } } ignore_live_leader: false dest_uuid: "eb664add87f44013b596f4cc01c9a41c" is_pre_election: true
21:04:15.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:15.178471 24113 raft_consensus.cc:2468] T 00000000000000000000000000000000 P eb664add87f44013b596f4cc01c9a41c [term 1 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate c520077acf3341e29447c6c2b8d3f74c in term 1.
21:04:15.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:15.179000 23310 leader_election.cc:304] T 00000000000000000000000000000000 P c520077acf3341e29447c6c2b8d3f74c [CANDIDATE]: Term 4 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: c520077acf3341e29447c6c2b8d3f74c, eb664add87f44013b596f4cc01c9a41c; no voters: e9dd218ca2244f5a934aac2b7e58eb1d
21:04:15.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:15.179324 24172 raft_consensus.cc:2804] T 00000000000000000000000000000000 P c520077acf3341e29447c6c2b8d3f74c [term 3 FOLLOWER]: Leader pre-election won for term 4
21:04:15.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:15.179454 24172 raft_consensus.cc:493] T 00000000000000000000000000000000 P c520077acf3341e29447c6c2b8d3f74c [term 3 FOLLOWER]: Starting leader election (detected failure of leader e9dd218ca2244f5a934aac2b7e58eb1d)
21:04:15.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:15.179553 24172 raft_consensus.cc:3060] T 00000000000000000000000000000000 P c520077acf3341e29447c6c2b8d3f74c [term 3 FOLLOWER]: Advancing to term 4
21:04:15.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:15.181797 24172 raft_consensus.cc:515] T 00000000000000000000000000000000 P c520077acf3341e29447c6c2b8d3f74c [term 4 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e9dd218ca2244f5a934aac2b7e58eb1d" member_type: VOTER last_known_addr { host: "127.22.168.62" port: 37239 } } peers { permanent_uuid: "c520077acf3341e29447c6c2b8d3f74c" member_type: VOTER last_known_addr { host: "127.22.168.61" port: 32947 } } peers { permanent_uuid: "eb664add87f44013b596f4cc01c9a41c" member_type: VOTER last_known_addr { host: "127.22.168.60" port: 33247 } }
21:04:15.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:15.182443 24172 leader_election.cc:290] T 00000000000000000000000000000000 P c520077acf3341e29447c6c2b8d3f74c [CANDIDATE]: Term 4 election: Requested vote from peers e9dd218ca2244f5a934aac2b7e58eb1d (127.22.168.62:37239), eb664add87f44013b596f4cc01c9a41c (127.22.168.60:33247)
21:04:15.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:15.183036 24113 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "c520077acf3341e29447c6c2b8d3f74c" candidate_term: 4 candidate_status { last_received { term: 3 index: 12 } } ignore_live_leader: false dest_uuid: "eb664add87f44013b596f4cc01c9a41c"
21:04:15.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:15.183248 24113 raft_consensus.cc:3060] T 00000000000000000000000000000000 P eb664add87f44013b596f4cc01c9a41c [term 1 FOLLOWER]: Advancing to term 4
21:04:15.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:15.183976 23307 leader_election.cc:336] T 00000000000000000000000000000000 P c520077acf3341e29447c6c2b8d3f74c [CANDIDATE]: Term 4 election: RPC error from VoteRequest() call to peer e9dd218ca2244f5a934aac2b7e58eb1d (127.22.168.62:37239): Network error: Client connection negotiation failed: client connection to 127.22.168.62:37239: connect: Connection refused (error 111)
21:04:15.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:15.187513 24113 raft_consensus.cc:2468] T 00000000000000000000000000000000 P eb664add87f44013b596f4cc01c9a41c [term 4 FOLLOWER]: Leader election vote request: Granting yes vote for candidate c520077acf3341e29447c6c2b8d3f74c in term 4.
21:04:15.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:15.187942 23310 leader_election.cc:304] T 00000000000000000000000000000000 P c520077acf3341e29447c6c2b8d3f74c [CANDIDATE]: Term 4 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: c520077acf3341e29447c6c2b8d3f74c, eb664add87f44013b596f4cc01c9a41c; no voters: e9dd218ca2244f5a934aac2b7e58eb1d
21:04:15.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:15.188278 24172 raft_consensus.cc:2804] T 00000000000000000000000000000000 P c520077acf3341e29447c6c2b8d3f74c [term 4 FOLLOWER]: Leader election won for term 4
21:04:15.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:15.188831 24172 raft_consensus.cc:697] T 00000000000000000000000000000000 P c520077acf3341e29447c6c2b8d3f74c [term 4 LEADER]: Becoming Leader. State: Replica: c520077acf3341e29447c6c2b8d3f74c, State: Running, Role: LEADER
21:04:15.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:15.189299 24172 consensus_queue.cc:237] T 00000000000000000000000000000000 P c520077acf3341e29447c6c2b8d3f74c [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 12, Committed index: 12, Last appended: 3.12, Last appended by leader: 12, Current term: 4, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e9dd218ca2244f5a934aac2b7e58eb1d" member_type: VOTER last_known_addr { host: "127.22.168.62" port: 37239 } } peers { permanent_uuid: "c520077acf3341e29447c6c2b8d3f74c" member_type: VOTER last_known_addr { host: "127.22.168.61" port: 32947 } } peers { permanent_uuid: "eb664add87f44013b596f4cc01c9a41c" member_type: VOTER last_known_addr { host: "127.22.168.60" port: 33247 } }
21:04:15.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:15.191053 24176 sys_catalog.cc:455] T 00000000000000000000000000000000 P c520077acf3341e29447c6c2b8d3f74c [sys.catalog]: SysCatalogTable state changed. Reason: New leader c520077acf3341e29447c6c2b8d3f74c. Latest consensus state: current_term: 4 leader_uuid: "c520077acf3341e29447c6c2b8d3f74c" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e9dd218ca2244f5a934aac2b7e58eb1d" member_type: VOTER last_known_addr { host: "127.22.168.62" port: 37239 } } peers { permanent_uuid: "c520077acf3341e29447c6c2b8d3f74c" member_type: VOTER last_known_addr { host: "127.22.168.61" port: 32947 } } peers { permanent_uuid: "eb664add87f44013b596f4cc01c9a41c" member_type: VOTER last_known_addr { host: "127.22.168.60" port: 33247 } } }
21:04:15.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:15.191349 24176 sys_catalog.cc:458] T 00000000000000000000000000000000 P c520077acf3341e29447c6c2b8d3f74c [sys.catalog]: This master's current role is: LEADER
21:04:15.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:15.192040 24178 catalog_manager.cc:1485] Loading table and tablet metadata into memory...
21:04:15.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:15.194561 24178 catalog_manager.cc:679] Loaded metadata for table txn_manager_fallback_rolling [id=03fcdef52a6f4a7099b05a3d2fa91c76]
21:04:15.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:15.195071 24178 catalog_manager.cc:679] Loaded metadata for table kudu_system.kudu_transactions [id=c54eb72594a5488d8dd9c9882da41ff4]
21:04:15.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:15.196918 24178 tablet_loader.cc:96] loaded metadata for tablet 01aa0e192c434b468ec7ed90d48563b3 (table txn_manager_fallback_rolling [id=03fcdef52a6f4a7099b05a3d2fa91c76])
21:04:15.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:15.197319 24178 tablet_loader.cc:96] loaded metadata for tablet 260c684b258041bab84c3ff36578f96b (table txn_manager_fallback_rolling [id=03fcdef52a6f4a7099b05a3d2fa91c76])
21:04:15.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:15.197641 24178 tablet_loader.cc:96] loaded metadata for tablet 581dbe2f34ca4a7981755fa5f4b6b6ca (table kudu_system.kudu_transactions [id=c54eb72594a5488d8dd9c9882da41ff4])
21:04:15.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:15.197922 24178 catalog_manager.cc:1494] Initializing Kudu cluster ID...
21:04:15.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:15.198751 24178 catalog_manager.cc:1269] Loaded cluster ID: 43f2ae537b96433b8f15997b1c5c1cd1
21:04:15.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:15.198849 24178 catalog_manager.cc:1505] Initializing Kudu internal certificate authority...
21:04:15.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:15.199968 24178 catalog_manager.cc:1514] Loading token signing keys...
21:04:15.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:15.200815 24178 catalog_manager.cc:6033] T 00000000000000000000000000000000 P c520077acf3341e29447c6c2b8d3f74c: Loaded TSK: 0
21:04:15.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:15.201710 24178 catalog_manager.cc:1524] Initializing in-progress tserver states...
21:04:15.224 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:15.226 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:15.226 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:15.226 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:15.228 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet Kudu Master for table 'Kudu Master' with partition [<start>, <end>)
21:04:15.229 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=KeepTransactionAlive, tablet=Kudu Master, attempt=8, TimeoutTracker(timeout=250, elapsed=244), Traces: [0ms] sending RPC to server master-127.22.168.62:37239, [1ms] received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [1ms] delaying RPC due to: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [19ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [23ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: OK, [39ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [39ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [66ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [66ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: OK, [66ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: OK, [83ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [83ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [103ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [123ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [123ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: OK, [123ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [123ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [162ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [163ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [164ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: OK, [164ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [242ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [243ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [244ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: OK, deferred=Deferred@65644512(state=PENDING, result=null, callback=org.apache.kudu.client.KuduTransaction$$Lambda$119/48695226@bd75073 -> (continuation of Deferred@737567292 after retry RPC after error@1711091688) -> (continuation of Deferred@728056336 after retry RPC after error@444648391) -> (continuation of Deferred@41921833 after retry RPC after error@250691809) -> (continuation of Deferred@252049133 after retry RPC after error@331860537) -> (continuation of Deferred@1814965133 after retry RPC after error@1750898849) -> (continuation of Deferred@1213114841 after retry RPC after error@1702180075), errback=org.apache.kudu.client.KuduTransaction$$Lambda$120/518121423@781a198b -> (continuation of Deferred@737567292 after retry RPC after error@1711091688) -> (continuation of Deferred@728056336 after retry RPC after error@444648391) -> (continuation of Deferred@41921833 after retry RPC after error@250691809) -> (continuation of Deferred@252049133 after retry RPC after error@331860537) -> (continuation of Deferred@1814965133 after retry RPC after error@1750898849) -> (continuation of Deferred@1213114841 after retry RPC after error@1702180075))) after lookup
21:04:15.245 [DEBUG - pool-40-thread-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=Kudu Master, attempt=9, TimeoutTracker(timeout=250, elapsed=260), Traces: [0ms] sending RPC to server master-127.22.168.62:37239, [1ms] received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [1ms] delaying RPC due to: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [19ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [23ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: OK, [39ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [39ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [66ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [66ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: OK, [66ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: OK, [83ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [83ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [103ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [123ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [123ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: OK, [123ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [123ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [162ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [163ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [164ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: OK, [164ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [242ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [243ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [244ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: OK, [245ms] sending RPC to server master-127.22.168.61:32947, deferred=Deferred@65644512(state=PENDING, result=null, callback=org.apache.kudu.client.KuduTransaction$$Lambda$119/48695226@bd75073 -> (continuation of Deferred@737567292 after retry RPC after error@1711091688) -> (continuation of Deferred@728056336 after retry RPC after error@444648391) -> (continuation of Deferred@41921833 after retry RPC after error@250691809) -> (continuation of Deferred@252049133 after retry RPC after error@331860537) -> (continuation of Deferred@1814965133 after retry RPC after error@1750898849) -> (continuation of Deferred@1213114841 after retry RPC after error@1702180075) -> (continuation of Deferred@1420420648 after retry RPC@1688129948), errback=org.apache.kudu.client.KuduTransaction$$Lambda$120/518121423@781a198b -> (continuation of Deferred@737567292 after retry RPC after error@1711091688) -> (continuation of Deferred@728056336 after retry RPC after error@444648391) -> (continuation of Deferred@41921833 after retry RPC after error@250691809) -> (continuation of Deferred@252049133 after retry RPC after error@331860537) -> (continuation of Deferred@1814965133 after retry RPC after error@1750898849) -> (continuation of Deferred@1213114841 after retry RPC after error@1702180075) -> (continuation of Deferred@1420420648 after retry RPC@1688129948)))
21:04:15.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:15.248867 23353 rpcz_store.cc:267] Call kudu.transactions.TxnManagerService.KeepTransactionAlive from 127.0.0.1:43940 (request call id 19) took 18 ms (client timeout 5 ms). Trace:
21:04:15.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:15.249176 23353 rpcz_store.cc:269] 1028 21:04:15.230268 (+ 0us) service_pool.cc:168] Inserting onto call queue
21:04:15.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1028 21:04:15.230346 (+ 78us) service_pool.cc:225] Handling call
21:04:15.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1028 21:04:15.248767 (+ 18421us) inbound_call.cc:173] Queueing success response
21:04:15.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"init.queue_time_us":544,"thread_start_us":782,"threads_started":3}
21:04:15.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/monotime.cc:332:28: runtime error: signed integer overflow: 896750963866 + 9223371140103813692 cannot be represented in type 'long'
21:04:15.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:15.278482 24113 raft_consensus.cc:1275] T 00000000000000000000000000000000 P eb664add87f44013b596f4cc01c9a41c [term 4 FOLLOWER]: Refusing update from remote peer c520077acf3341e29447c6c2b8d3f74c: Log matching property violated. Preceding OpId in replica: term: 1 index: 11. Preceding OpId from leader: term: 4 index: 13. (index mismatch)
21:04:15.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:15.279904 24176 consensus_queue.cc:1048] T 00000000000000000000000000000000 P c520077acf3341e29447c6c2b8d3f74c [LEADER]: Connected to new peer: Peer: permanent_uuid: "eb664add87f44013b596f4cc01c9a41c" member_type: VOTER last_known_addr { host: "127.22.168.60" port: 33247 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 13, Last known committed idx: 11, Time since last communication: 0.000s
21:04:15.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:15.272915 23323 catalog_manager.cc:2257] Servicing CreateTable request from {username='slave'} at 127.0.0.1:43984:
21:04:15.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "kudu_system.kudu_transactions"
21:04:15.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) schema {
21:04:15.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
21:04:15.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_id"
21:04:15.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT64
21:04:15.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
21:04:15.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
21:04:15.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
21:04:15.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
21:04:15.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
21:04:15.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
21:04:15.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
21:04:15.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
21:04:15.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "entry_type"
21:04:15.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT8
21:04:15.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
21:04:15.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
21:04:15.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
21:04:15.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
21:04:15.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
21:04:15.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
21:04:15.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
21:04:15.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
21:04:15.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "identifier"
21:04:15.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
21:04:15.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
21:04:15.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
21:04:15.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
21:04:15.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
21:04:15.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
21:04:15.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
21:04:15.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
21:04:15.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
21:04:15.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "metadata"
21:04:15.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
21:04:15.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
21:04:15.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
21:04:15.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
21:04:15.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
21:04:15.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
21:04:15.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
21:04:15.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
21:04:15.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
21:04:15.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) num_replicas: 3
21:04:15.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) split_rows_range_bounds {
21:04:15.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) rows: "<redacted>""\006\001\000\000\000\000\000\000\000\000\007\001@B\017\000\000\000\000\000"
21:04:15.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) indirect_data: "<redacted>"""
21:04:15.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
21:04:15.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) partition_schema {
21:04:15.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) range_schema {
21:04:15.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
21:04:15.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_id"
21:04:15.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
21:04:15.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
21:04:15.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
21:04:15.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) table_type: TXN_STATUS_TABLE
21:04:15.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:15.292881 23323 catalog_manager.cc:7011] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table kudu_system.kudu_transactions in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
21:04:15.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:15.294695 23307 consensus_peers.cc:597] T 00000000000000000000000000000000 P c520077acf3341e29447c6c2b8d3f74c -> Peer e9dd218ca2244f5a934aac2b7e58eb1d (127.22.168.62:37239): Couldn't send request to peer e9dd218ca2244f5a934aac2b7e58eb1d. Status: Network error: Client connection negotiation failed: client connection to 127.22.168.62:37239: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
21:04:15.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:15.302383 23653 heartbeater.cc:344] Connected to a master server at 127.22.168.60:33247
21:04:15.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:15.306921 24191 sys_catalog.cc:455] T 00000000000000000000000000000000 P eb664add87f44013b596f4cc01c9a41c [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 4 leader_uuid: "c520077acf3341e29447c6c2b8d3f74c" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e9dd218ca2244f5a934aac2b7e58eb1d" member_type: VOTER last_known_addr { host: "127.22.168.62" port: 37239 } } peers { permanent_uuid: "c520077acf3341e29447c6c2b8d3f74c" member_type: VOTER last_known_addr { host: "127.22.168.61" port: 32947 } } peers { permanent_uuid: "eb664add87f44013b596f4cc01c9a41c" member_type: VOTER last_known_addr { host: "127.22.168.60" port: 33247 } } }
21:04:15.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:15.307355 24191 sys_catalog.cc:458] T 00000000000000000000000000000000 P eb664add87f44013b596f4cc01c9a41c [sys.catalog]: This master's current role is: FOLLOWER
21:04:15.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:15.308507 24152 sys_catalog.cc:455] T 00000000000000000000000000000000 P eb664add87f44013b596f4cc01c9a41c [sys.catalog]: SysCatalogTable state changed. Reason: New leader c520077acf3341e29447c6c2b8d3f74c. Latest consensus state: current_term: 4 leader_uuid: "c520077acf3341e29447c6c2b8d3f74c" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e9dd218ca2244f5a934aac2b7e58eb1d" member_type: VOTER last_known_addr { host: "127.22.168.62" port: 37239 } } peers { permanent_uuid: "c520077acf3341e29447c6c2b8d3f74c" member_type: VOTER last_known_addr { host: "127.22.168.61" port: 32947 } } peers { permanent_uuid: "eb664add87f44013b596f4cc01c9a41c" member_type: VOTER last_known_addr { host: "127.22.168.60" port: 33247 } } }
21:04:15.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:15.308753 24152 sys_catalog.cc:458] T 00000000000000000000000000000000 P eb664add87f44013b596f4cc01c9a41c [sys.catalog]: This master's current role is: FOLLOWER
21:04:15.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:15.309056 24102 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "210cdae6f0e44d83acf5641ce563b738" instance_seqno: 1761685450340693) as {username='slave'} at 127.22.168.1:32925; Asking this server to re-register.
21:04:15.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:15.309901 23653 heartbeater.cc:461] Registering TS with master...
21:04:15.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:15.311666 24102 ts_manager.cc:194] Registered new tserver with Master: 210cdae6f0e44d83acf5641ce563b738 (127.22.168.1:42241)
21:04:15.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:15.321202 24152 sys_catalog.cc:455] T 00000000000000000000000000000000 P eb664add87f44013b596f4cc01c9a41c [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 4 leader_uuid: "c520077acf3341e29447c6c2b8d3f74c" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e9dd218ca2244f5a934aac2b7e58eb1d" member_type: VOTER last_known_addr { host: "127.22.168.62" port: 37239 } } peers { permanent_uuid: "c520077acf3341e29447c6c2b8d3f74c" member_type: VOTER last_known_addr { host: "127.22.168.61" port: 32947 } } peers { permanent_uuid: "eb664add87f44013b596f4cc01c9a41c" member_type: VOTER last_known_addr { host: "127.22.168.60" port: 33247 } } }
21:04:15.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:15.321452 24152 sys_catalog.cc:458] T 00000000000000000000000000000000 P eb664add87f44013b596f4cc01c9a41c [sys.catalog]: This master's current role is: FOLLOWER
21:04:15.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:15.322393 24172 sys_catalog.cc:455] T 00000000000000000000000000000000 P c520077acf3341e29447c6c2b8d3f74c [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 4 leader_uuid: "c520077acf3341e29447c6c2b8d3f74c" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e9dd218ca2244f5a934aac2b7e58eb1d" member_type: VOTER last_known_addr { host: "127.22.168.62" port: 37239 } } peers { permanent_uuid: "c520077acf3341e29447c6c2b8d3f74c" member_type: VOTER last_known_addr { host: "127.22.168.61" port: 32947 } } peers { permanent_uuid: "eb664add87f44013b596f4cc01c9a41c" member_type: VOTER last_known_addr { host: "127.22.168.60" port: 33247 } } }
21:04:15.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:15.324350 24172 sys_catalog.cc:458] T 00000000000000000000000000000000 P c520077acf3341e29447c6c2b8d3f74c [sys.catalog]: This master's current role is: LEADER
21:04:15.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:15.710081 23794 heartbeater.cc:344] Connected to a master server at 127.22.168.60:33247
21:04:15.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:15.712210 24102 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "1acec06b763242d082ddb75231fc4eca" instance_seqno: 1761685450813816) as {username='slave'} at 127.22.168.2:44499; Asking this server to re-register.
21:04:15.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:15.712879 23794 heartbeater.cc:461] Registering TS with master...
21:04:15.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:15.714072 24102 ts_manager.cc:194] Registered new tserver with Master: 1acec06b763242d082ddb75231fc4eca (127.22.168.2:37409)
21:04:15.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:15.824148 23307 consensus_peers.cc:597] T 00000000000000000000000000000000 P c520077acf3341e29447c6c2b8d3f74c -> Peer e9dd218ca2244f5a934aac2b7e58eb1d (127.22.168.62:37239): Couldn't send request to peer e9dd218ca2244f5a934aac2b7e58eb1d. Status: Network error: Client connection negotiation failed: client connection to 127.22.168.62:37239: connect: Connection refused (error 111). This is attempt 6: this message will repeat every 5th retry.
21:04:16.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:16.285938 23655 heartbeater.cc:499] Master 127.22.168.61:32947 was elected leader, sending a full tablet report...
21:04:16.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:16.358986 23307 consensus_peers.cc:597] T 00000000000000000000000000000000 P c520077acf3341e29447c6c2b8d3f74c -> Peer e9dd218ca2244f5a934aac2b7e58eb1d (127.22.168.62:37239): Couldn't send request to peer e9dd218ca2244f5a934aac2b7e58eb1d. Status: Network error: Client connection negotiation failed: client connection to 127.22.168.62:37239: connect: Connection refused (error 111). This is attempt 11: this message will repeat every 5th retry.
21:04:16.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #0 0x7f6a14326a28 in kudu::operator+(kudu::MonoTime const&, kudu::MonoDelta const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/monotime.cc:332:28
21:04:16.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #1 0xa77c65 in kudu::CountDownLatch::WaitFor(kudu::MonoDelta const&) const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/countdown_latch.h:99:38
21:04:16.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #2 0x7f6a2807cec2 in kudu::Promise<kudu::Status>::WaitFor(kudu::MonoDelta const&) const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/promise.h:57:16
21:04:16.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #3 0x7f6a2806c9d4 in kudu::master::Master::WaitForTxnManagerInit(kudu::MonoTime) const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/master/master.cc:482:48
21:04:16.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #4 0x7f6a281d118e in kudu::transactions::TxnManager::CheckInitialized(kudu::MonoTime const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/master/txn_manager.cc:283:21
21:04:16.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #5 0x7f6a281d1d6b in kudu::transactions::TxnManager::KeepTransactionAlive(long, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, kudu::MonoTime const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/master/txn_manager.cc:226:3
21:04:16.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #6 0x7f6a281d69af in kudu::transactions::TxnManagerServiceImpl::KeepTransactionAlive(kudu::transactions::KeepTransactionAliveRequestPB const*, kudu::transactions::KeepTransactionAliveResponsePB*, kudu::rpc::RpcContext*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/master/txn_manager_service.cc:159:42
21:04:16.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #7 0x7f6a220062d6 in kudu::transactions::TxnManagerServiceIf::TxnManagerServiceIf(scoped_refptr<kudu::MetricEntity> const&, scoped_refptr<kudu::rpc::ResultTracker> const&)::$_9::operator()(google::protobuf::Message const*, google::protobuf::Message*, kudu::rpc::RpcContext*) const /home/jenkins-slave/workspace/build_and_test_flaky@2/build/asan/src/kudu/master/txn_manager.service.cc:271:13
21:04:16.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #8 0x7f6a22006005 in std::_Function_handler<void (google::protobuf::Message const*, google::protobuf::Message*, kudu::rpc::RpcContext*), kudu::transactions::TxnManagerServiceIf::TxnManagerServiceIf(scoped_refptr<kudu::MetricEntity> const&, scoped_refptr<kudu::rpc::ResultTracker> const&)::$_9>::_M_invoke(std::_Any_data const&, google::protobuf::Message const*&&, google::protobuf::Message*&&, kudu::rpc::RpcContext*&&) ../../../include/c++/7.5.0/bits/std_function.h:316:2
21:04:16.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #9 0x7f6a1b37027e in std::function<void (google::protobuf::Message const*, google::protobuf::Message*, kudu::rpc::RpcContext*)>::operator()(google::protobuf::Message const*, google::protobuf::Message*, kudu::rpc::RpcContext*) const ../../../include/c++/7.5.0/bits/std_function.h:706:14
21:04:16.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #10 0x7f6a1b36f15e in kudu::rpc::GeneratedServiceIf::Handle(kudu::rpc::InboundCall*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/rpc/service_if.cc:137:3
21:04:16.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #11 0x7f6a1b375395 in kudu::rpc::ServicePool::RunThread() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/rpc/service_pool.cc:229:15
21:04:16.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #12 0x7f6a1b375cbf in kudu::rpc::ServicePool::Init(int)::$_0::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/rpc/service_pool.cc:92:5
21:04:16.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #13 0x7f6a1b375ae8 in std::_Function_handler<void (), kudu::rpc::ServicePool::Init(int)::$_0>::_M_invoke(std::_Any_data const&) ../../../include/c++/7.5.0/bits/std_function.h:316:2
21:04:16.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #14 0xa8867c in std::function<void ()>::operator()() const ../../../include/c++/7.5.0/bits/std_function.h:706:14
21:04:16.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #15 0x7f6a143f6ee8 in kudu::Thread::SuperviseThread(void*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:694:3
21:04:16.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #16 0x7f6a17a016da in start_thread (/lib/x86_64-linux-gnu/libpthread.so.0+0x76da)
21:04:16.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #17 0x7f6a1053371e in clone (/lib/x86_64-linux-gnu/libc.so.6+0x12171e)
21:04:16.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
21:04:16.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) SUMMARY: UndefinedBehaviorSanitizer: undefined-behavior /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/monotime.cc:332:28 in
21:04:16.562 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.22.168.61:32947(127.22.168.61:32947)] handling channelInactive
21:04:16.562 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state READY due to: connection closed
21:04:16.562 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location master-127.22.168.61:32947(127.22.168.61:32947) for tablet Kudu Master: connection closed
21:04:16.565 [WARN - pool-40-thread-1] (HashedWheelTimer.java:716) An exception was thrown by TimerTask.
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343) ~[main/:?]
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at java.lang.Thread.run(Thread.java:748) [?:1.8.0_275]
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86)
at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106)
at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198)
at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158)
at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953)
at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923)
at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464)
at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713)
at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701)
at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788)
at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501)
at java.lang.Thread.run(Thread.java:748)
21:04:16.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:16.706813 23796 heartbeater.cc:499] Master 127.22.168.61:32947 was elected leader, sending a full tablet report...
21:04:16.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:16.781566 23938 heartbeater.cc:499] Master 127.22.168.61:32947 was elected leader, sending a full tablet report...
21:04:16.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:16.817991 24204 raft_consensus.cc:493] T 00000000000000000000000000000000 P eb664add87f44013b596f4cc01c9a41c [term 4 FOLLOWER]: Starting pre-election (detected failure of leader c520077acf3341e29447c6c2b8d3f74c)
21:04:16.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:16.818202 24204 raft_consensus.cc:515] T 00000000000000000000000000000000 P eb664add87f44013b596f4cc01c9a41c [term 4 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e9dd218ca2244f5a934aac2b7e58eb1d" member_type: VOTER last_known_addr { host: "127.22.168.62" port: 37239 } } peers { permanent_uuid: "c520077acf3341e29447c6c2b8d3f74c" member_type: VOTER last_known_addr { host: "127.22.168.61" port: 32947 } } peers { permanent_uuid: "eb664add87f44013b596f4cc01c9a41c" member_type: VOTER last_known_addr { host: "127.22.168.60" port: 33247 } }
21:04:16.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:16.818943 24204 leader_election.cc:290] T 00000000000000000000000000000000 P eb664add87f44013b596f4cc01c9a41c [CANDIDATE]: Term 5 pre-election: Requested pre-vote from peers e9dd218ca2244f5a934aac2b7e58eb1d (127.22.168.62:37239), c520077acf3341e29447c6c2b8d3f74c (127.22.168.61:32947)
21:04:16.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:16.821364 24086 leader_election.cc:336] T 00000000000000000000000000000000 P eb664add87f44013b596f4cc01c9a41c [CANDIDATE]: Term 5 pre-election: RPC error from VoteRequest() call to peer e9dd218ca2244f5a934aac2b7e58eb1d (127.22.168.62:37239): Network error: Client connection negotiation failed: client connection to 127.22.168.62:37239: connect: Connection refused (error 111)
21:04:16.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:16.821581 24089 leader_election.cc:336] T 00000000000000000000000000000000 P eb664add87f44013b596f4cc01c9a41c [CANDIDATE]: Term 5 pre-election: RPC error from VoteRequest() call to peer c520077acf3341e29447c6c2b8d3f74c (127.22.168.61:32947): Network error: Client connection negotiation failed: client connection to 127.22.168.61:32947: connect: Connection refused (error 111)
21:04:16.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:16.821700 24089 leader_election.cc:304] T 00000000000000000000000000000000 P eb664add87f44013b596f4cc01c9a41c [CANDIDATE]: Term 5 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: eb664add87f44013b596f4cc01c9a41c; no voters: c520077acf3341e29447c6c2b8d3f74c, e9dd218ca2244f5a934aac2b7e58eb1d
21:04:16.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:16.821972 24204 raft_consensus.cc:2749] T 00000000000000000000000000000000 P eb664add87f44013b596f4cc01c9a41c [term 4 FOLLOWER]: Leader pre-election lost for term 5. Reason: could not achieve majority
21:04:16.904 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:16.905 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:16.905 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:16.905 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:16.906 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:16.906 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:16.906 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:16.906 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:16.906 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:16.907 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:16.907 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:16.907 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:16.907 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:16.907 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:16.907 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:16.908 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:16.908 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:16.909 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:16.925 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:16.925 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:16.926 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:16.926 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:16.926 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:16.927 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:16.927 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:16.927 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:16.927 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:16.928 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:16.928 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:16.928 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:16.928 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:16.944 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:16.944 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:16.945 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:16.945 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:16.945 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:16.945 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:16.946 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:16.946 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:16.946 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:16.946 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:16.947 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:16.947 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:16.964 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:16.965 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:16.966 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:16.966 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:16.966 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:16.966 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:16.966 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:16.966 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:16.967 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:16.967 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:16.967 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:16.967 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:16.985 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:16.985 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:16.986 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:16.986 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:16.986 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:16.986 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:16.987 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:16.987 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:16.987 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:16.987 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:16.988 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:16.988 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:17.004 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:17.005 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:17.005 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:17.005 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:17.006 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:17.006 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:17.006 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:17.006 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:17.006 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:17.006 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:17.007 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:17.007 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:17.025 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:17.025 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:17.026 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:17.026 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:17.026 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:17.026 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:17.026 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:17.027 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:17.027 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:17.027 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:17.027 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:17.028 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:17.064 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:17.065 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:17.066 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:17.066 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:17.066 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:17.066 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:17.066 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:17.066 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:17.067 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:17.067 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=163), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [20ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [23ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [23ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [24ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [41ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [43ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [63ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [80ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [83ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [83ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [84ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [120ms] refreshing cache from master, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [123ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [162ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [162ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [163ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [163ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, deferred=Deferred@214064826(state=PENDING, result=null, callback=(continuation of Deferred@1259954511 after retry RPC after error@1294657982) -> (continuation of Deferred@664263862 after retry RPC after error@960389488) -> (continuation of Deferred@959282177 after retry RPC after error@600839555) -> (continuation of Deferred@976949027 after retry RPC after error@1883557109) -> (continuation of Deferred@576416648 after retry RPC after error@25435725) -> (continuation of Deferred@1671715657 after retry RPC after error@894501036) -> (continuation of Deferred@1286548133 after retry RPC after error@1836593931), errback=(continuation of Deferred@1259954511 after retry RPC after error@1294657982) -> (continuation of Deferred@664263862 after retry RPC after error@960389488) -> (continuation of Deferred@959282177 after retry RPC after error@600839555) -> (continuation of Deferred@976949027 after retry RPC after error@1883557109) -> (continuation of Deferred@576416648 after retry RPC after error@25435725) -> (continuation of Deferred@1671715657 after retry RPC after error@894501036) -> (continuation of Deferred@1286548133 after retry RPC after error@1836593931)))
21:04:17.068 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=163), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [20ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [23ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [23ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [24ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [41ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [43ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [63ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [80ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [83ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [83ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [84ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [120ms] refreshing cache from master, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [123ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [162ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [162ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [163ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [163ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, deferred=Deferred@214064826(state=PENDING, result=null, callback=(continuation of Deferred@1259954511 after retry RPC after error@1294657982) -> (continuation of Deferred@664263862 after retry RPC after error@960389488) -> (continuation of Deferred@959282177 after retry RPC after error@600839555) -> (continuation of Deferred@976949027 after retry RPC after error@1883557109) -> (continuation of Deferred@576416648 after retry RPC after error@25435725) -> (continuation of Deferred@1671715657 after retry RPC after error@894501036) -> (continuation of Deferred@1286548133 after retry RPC after error@1836593931), errback=(continuation of Deferred@1259954511 after retry RPC after error@1294657982) -> (continuation of Deferred@664263862 after retry RPC after error@960389488) -> (continuation of Deferred@959282177 after retry RPC after error@600839555) -> (continuation of Deferred@976949027 after retry RPC after error@1883557109) -> (continuation of Deferred@576416648 after retry RPC after error@25435725) -> (continuation of Deferred@1671715657 after retry RPC after error@894501036) -> (continuation of Deferred@1286548133 after retry RPC after error@1836593931)))
21:04:17.084 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:17.085 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:17.086 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:17.086 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:17.086 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:17.086 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:17.086 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:17.086 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:17.086 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:17.086 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:17.087 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:17.087 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:17.104 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:17.104 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:17.105 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:17.105 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:17.105 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:17.105 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:17.105 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:17.105 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:17.106 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:17.124 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:17.125 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:17.125 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:17.125 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:17.126 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:17.126 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:17.126 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:17.126 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:17.127 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:17.144 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:17.145 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:17.146 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:17.146 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:17.146 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:17.146 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:17.146 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:17.146 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:17.147 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:17.147 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:17.147 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:17.148 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:17.164 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:17.165 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:17.165 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:17.165 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:17.165 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:17.166 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:17.166 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:17.166 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:17.167 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:17.184 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:17.185 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:17.186 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:17.186 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:17.186 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:17.186 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:17.186 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:17.186 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:17.186 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:17.186 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:17.187 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:17.187 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:17.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:17.225867 24204 raft_consensus.cc:493] T 00000000000000000000000000000000 P eb664add87f44013b596f4cc01c9a41c [term 4 FOLLOWER]: Starting pre-election (detected failure of leader c520077acf3341e29447c6c2b8d3f74c)
21:04:17.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:17.226089 24204 raft_consensus.cc:515] T 00000000000000000000000000000000 P eb664add87f44013b596f4cc01c9a41c [term 4 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e9dd218ca2244f5a934aac2b7e58eb1d" member_type: VOTER last_known_addr { host: "127.22.168.62" port: 37239 } } peers { permanent_uuid: "c520077acf3341e29447c6c2b8d3f74c" member_type: VOTER last_known_addr { host: "127.22.168.61" port: 32947 } } peers { permanent_uuid: "eb664add87f44013b596f4cc01c9a41c" member_type: VOTER last_known_addr { host: "127.22.168.60" port: 33247 } }
21:04:17.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:17.226756 24204 leader_election.cc:290] T 00000000000000000000000000000000 P eb664add87f44013b596f4cc01c9a41c [CANDIDATE]: Term 5 pre-election: Requested pre-vote from peers e9dd218ca2244f5a934aac2b7e58eb1d (127.22.168.62:37239), c520077acf3341e29447c6c2b8d3f74c (127.22.168.61:32947)
21:04:17.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:17.228281 24089 leader_election.cc:336] T 00000000000000000000000000000000 P eb664add87f44013b596f4cc01c9a41c [CANDIDATE]: Term 5 pre-election: RPC error from VoteRequest() call to peer c520077acf3341e29447c6c2b8d3f74c (127.22.168.61:32947): Network error: Client connection negotiation failed: client connection to 127.22.168.61:32947: connect: Connection refused (error 111)
21:04:17.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:17.228482 24086 leader_election.cc:336] T 00000000000000000000000000000000 P eb664add87f44013b596f4cc01c9a41c [CANDIDATE]: Term 5 pre-election: RPC error from VoteRequest() call to peer e9dd218ca2244f5a934aac2b7e58eb1d (127.22.168.62:37239): Network error: Client connection negotiation failed: client connection to 127.22.168.62:37239: connect: Connection refused (error 111)
21:04:17.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:17.228644 24086 leader_election.cc:304] T 00000000000000000000000000000000 P eb664add87f44013b596f4cc01c9a41c [CANDIDATE]: Term 5 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: eb664add87f44013b596f4cc01c9a41c; no voters: c520077acf3341e29447c6c2b8d3f74c, e9dd218ca2244f5a934aac2b7e58eb1d
21:04:17.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:17.228981 24204 raft_consensus.cc:2749] T 00000000000000000000000000000000 P eb664add87f44013b596f4cc01c9a41c [term 4 FOLLOWER]: Leader pre-election lost for term 5. Reason: could not achieve majority
21:04:17.244 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:17.245 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:17.245 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:17.245 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:17.245 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:17.246 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:17.246 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:17.246 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:17.247 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:17.248 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=161), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [18ms] refreshing cache from master, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [20ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [20ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [21ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [39ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [63ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [79ms] refreshing cache from master, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [80ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [159ms] refreshing cache from master, [159ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [159ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [160ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [161ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [162ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, deferred=Deferred@1369518641(state=PENDING, result=null, callback=(continuation of Deferred@2056809404 after retry RPC after error@336456035) -> (continuation of Deferred@1977839815 after retry RPC after error@755754224) -> (continuation of Deferred@343388006 after retry RPC after error@1875511930) -> (continuation of Deferred@21047820 after retry RPC after error@1931440816) -> (continuation of Deferred@1513815850 after retry RPC after error@1704765389) -> (continuation of Deferred@1829101624 after retry RPC after error@1703384538), errback=(continuation of Deferred@2056809404 after retry RPC after error@336456035) -> (continuation of Deferred@1977839815 after retry RPC after error@755754224) -> (continuation of Deferred@343388006 after retry RPC after error@1875511930) -> (continuation of Deferred@21047820 after retry RPC after error@1931440816) -> (continuation of Deferred@1513815850 after retry RPC after error@1704765389) -> (continuation of Deferred@1829101624 after retry RPC after error@1703384538)))
21:04:17.249 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=161), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [18ms] refreshing cache from master, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [20ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [20ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [21ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [39ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [63ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [79ms] refreshing cache from master, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [80ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [159ms] refreshing cache from master, [159ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [159ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [160ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [161ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [162ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, deferred=Deferred@1369518641(state=PENDING, result=null, callback=(continuation of Deferred@2056809404 after retry RPC after error@336456035) -> (continuation of Deferred@1977839815 after retry RPC after error@755754224) -> (continuation of Deferred@343388006 after retry RPC after error@1875511930) -> (continuation of Deferred@21047820 after retry RPC after error@1931440816) -> (continuation of Deferred@1513815850 after retry RPC after error@1704765389) -> (continuation of Deferred@1829101624 after retry RPC after error@1703384538), errback=(continuation of Deferred@2056809404 after retry RPC after error@336456035) -> (continuation of Deferred@1977839815 after retry RPC after error@755754224) -> (continuation of Deferred@343388006 after retry RPC after error@1875511930) -> (continuation of Deferred@21047820 after retry RPC after error@1931440816) -> (continuation of Deferred@1513815850 after retry RPC after error@1704765389) -> (continuation of Deferred@1829101624 after retry RPC after error@1703384538)))
21:04:17.265 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:17.266 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:17.267 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:17.267 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:17.267 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:17.267 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:17.267 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:17.267 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:17.269 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:17.284 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:17.285 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:17.285 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:17.285 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:17.285 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:17.286 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:17.286 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:17.286 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:17.289 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:17.304 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:17.305 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:17.306 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:17.306 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:17.306 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:17.308 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:17.308 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:17.309 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:17.309 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:17.325 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:17.325 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:17.326 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:17.326 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:17.326 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:17.327 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:17.327 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:17.327 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:17.327 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:17.344 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:17.345 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:17.345 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:17.345 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:17.346 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:17.346 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:17.346 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:17.346 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:17.347 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:17.364 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:17.365 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:17.366 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:17.366 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:17.366 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:17.366 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:17.366 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:17.367 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:17.367 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:17.384 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:17.384 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:17.385 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:17.385 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:17.385 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:17.386 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:17.386 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:17.386 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:17.387 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:17.445 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:17.445 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:17.446 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:17.446 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:17.446 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:17.447 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:17.447 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:17.447 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:17.447 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:17.448 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=183), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [2ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [2ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [3ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [3ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [4ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [5ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [25ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [25ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [44ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [45ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [45ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [63ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [63ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [64ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [103ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [119ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [182ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [183ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [183ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [183ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, deferred=Deferred@719141576(state=PENDING, result=null, callback=(continuation of Deferred@1779937804 after retry RPC after error@638896527) -> (continuation of Deferred@1378683506 after retry RPC after error@300245381) -> (continuation of Deferred@1177892455 after retry RPC after error@1189992351) -> (continuation of Deferred@2083072938 after retry RPC after error@1190823139) -> (continuation of Deferred@697500757 after retry RPC after error@23752663) -> (continuation of Deferred@875257899 after retry RPC after error@644098001) -> (continuation of Deferred@400830180 after retry RPC after error@1168310040), errback=(continuation of Deferred@1779937804 after retry RPC after error@638896527) -> (continuation of Deferred@1378683506 after retry RPC after error@300245381) -> (continuation of Deferred@1177892455 after retry RPC after error@1189992351) -> (continuation of Deferred@2083072938 after retry RPC after error@1190823139) -> (continuation of Deferred@697500757 after retry RPC after error@23752663) -> (continuation of Deferred@875257899 after retry RPC after error@644098001) -> (continuation of Deferred@400830180 after retry RPC after error@1168310040)))
21:04:17.448 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=183), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [2ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [2ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [3ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [3ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [4ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [5ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [25ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [25ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [44ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [45ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [45ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [63ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [63ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [64ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [103ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [119ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [182ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [183ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [183ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [183ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, deferred=Deferred@719141576(state=PENDING, result=null, callback=(continuation of Deferred@1779937804 after retry RPC after error@638896527) -> (continuation of Deferred@1378683506 after retry RPC after error@300245381) -> (continuation of Deferred@1177892455 after retry RPC after error@1189992351) -> (continuation of Deferred@2083072938 after retry RPC after error@1190823139) -> (continuation of Deferred@697500757 after retry RPC after error@23752663) -> (continuation of Deferred@875257899 after retry RPC after error@644098001) -> (continuation of Deferred@400830180 after retry RPC after error@1168310040), errback=(continuation of Deferred@1779937804 after retry RPC after error@638896527) -> (continuation of Deferred@1378683506 after retry RPC after error@300245381) -> (continuation of Deferred@1177892455 after retry RPC after error@1189992351) -> (continuation of Deferred@2083072938 after retry RPC after error@1190823139) -> (continuation of Deferred@697500757 after retry RPC after error@23752663) -> (continuation of Deferred@875257899 after retry RPC after error@644098001) -> (continuation of Deferred@400830180 after retry RPC after error@1168310040)))
21:04:17.464 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:17.465 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:17.466 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:17.466 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:17.466 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:17.466 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:17.466 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:17.466 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:17.467 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:17.484 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:17.485 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:17.485 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:17.486 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:17.486 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:17.486 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:17.486 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:17.486 [INFO - pool-40-thread-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:17.487 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:17.505 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:17.505 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:17.506 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:17.506 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:17.506 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:17.507 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:17.507 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:17.507 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:17.507 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:17.524 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:17.525 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:17.526 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:17.526 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:17.526 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:17.526 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:17.526 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:17.526 [INFO - pool-40-thread-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:17.527 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:17.545 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:17.545 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:17.546 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:17.546 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:17.546 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:17.546 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:17.546 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:17.546 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:17.548 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:17.564 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:17.565 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:17.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:17.565845 23535 txn_status_manager.cc:1391] automatically aborted stale txn (ID 0) past 1.158s from last keepalive heartbeat (effective timeout is 1.000s)
21:04:17.566 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:17.566 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:17.566 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:17.567 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:17.567 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:17.567 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:17.569 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:17.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:17.585106 23569 master_proxy_rpc.cc:203] Re-attempting LookupRpcById request to leader Master (127.22.168.60:33247)
21:04:17.625 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:17.625 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:17.626 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:17.626 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:17.626 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:17.626 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:17.626 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:17.626 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:17.628 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:17.664 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:17.665 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:17.665 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:17.665 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:17.665 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:17.665 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:17.666 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:17.666 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:17.668 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:17.704 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:17.705 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:17.706 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:17.706 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:17.706 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:17.706 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:17.706 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:17.706 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:17.707 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:17.707 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=243), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [23ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [43ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [43ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [84ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [85ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [103ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [105ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [106ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [162ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [162ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [163ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [164ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [242ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [242ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [243ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [243ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, deferred=Deferred@1948535355(state=PENDING, result=null, callback=(continuation of Deferred@1582191901 after retry RPC after error@1413920706) -> (continuation of Deferred@1194211414 after retry RPC after error@286119706) -> (continuation of Deferred@1028099739 after retry RPC after error@1558103638) -> (continuation of Deferred@1778694419 after retry RPC after error@365827662) -> (continuation of Deferred@343695674 after retry RPC after error@1052440981) -> (continuation of Deferred@1907954544 after retry RPC after error@337647471) -> (continuation of Deferred@1671614816 after retry RPC after error@171829968), errback=(continuation of Deferred@1582191901 after retry RPC after error@1413920706) -> (continuation of Deferred@1194211414 after retry RPC after error@286119706) -> (continuation of Deferred@1028099739 after retry RPC after error@1558103638) -> (continuation of Deferred@1778694419 after retry RPC after error@365827662) -> (continuation of Deferred@343695674 after retry RPC after error@1052440981) -> (continuation of Deferred@1907954544 after retry RPC after error@337647471) -> (continuation of Deferred@1671614816 after retry RPC after error@171829968)))
21:04:17.708 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=243), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [23ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [43ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [43ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [84ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [85ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [103ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [105ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [106ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [162ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [162ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [163ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [164ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [242ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [242ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [243ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [243ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, deferred=Deferred@1948535355(state=PENDING, result=null, callback=(continuation of Deferred@1582191901 after retry RPC after error@1413920706) -> (continuation of Deferred@1194211414 after retry RPC after error@286119706) -> (continuation of Deferred@1028099739 after retry RPC after error@1558103638) -> (continuation of Deferred@1778694419 after retry RPC after error@365827662) -> (continuation of Deferred@343695674 after retry RPC after error@1052440981) -> (continuation of Deferred@1907954544 after retry RPC after error@337647471) -> (continuation of Deferred@1671614816 after retry RPC after error@171829968), errback=(continuation of Deferred@1582191901 after retry RPC after error@1413920706) -> (continuation of Deferred@1194211414 after retry RPC after error@286119706) -> (continuation of Deferred@1028099739 after retry RPC after error@1558103638) -> (continuation of Deferred@1778694419 after retry RPC after error@365827662) -> (continuation of Deferred@343695674 after retry RPC after error@1052440981) -> (continuation of Deferred@1907954544 after retry RPC after error@337647471) -> (continuation of Deferred@1671614816 after retry RPC after error@171829968)))
21:04:17.724 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:17.724 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:17.725 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:17.725 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:17.725 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:17.726 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:17.726 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:17.726 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:17.727 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:17.744 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:17.745 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:17.745 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:17.745 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:17.745 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:17.746 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:17.746 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:17.746 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:17.746 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:17.764 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:17.765 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:17.765 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:17.765 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:17.765 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:17.766 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:17.766 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:17.766 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:17.768 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:17.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:17.769042 24225 raft_consensus.cc:493] T 00000000000000000000000000000000 P eb664add87f44013b596f4cc01c9a41c [term 4 FOLLOWER]: Starting pre-election (detected failure of leader c520077acf3341e29447c6c2b8d3f74c)
21:04:17.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:17.769232 24225 raft_consensus.cc:515] T 00000000000000000000000000000000 P eb664add87f44013b596f4cc01c9a41c [term 4 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e9dd218ca2244f5a934aac2b7e58eb1d" member_type: VOTER last_known_addr { host: "127.22.168.62" port: 37239 } } peers { permanent_uuid: "c520077acf3341e29447c6c2b8d3f74c" member_type: VOTER last_known_addr { host: "127.22.168.61" port: 32947 } } peers { permanent_uuid: "eb664add87f44013b596f4cc01c9a41c" member_type: VOTER last_known_addr { host: "127.22.168.60" port: 33247 } }
21:04:17.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:17.769778 24225 leader_election.cc:290] T 00000000000000000000000000000000 P eb664add87f44013b596f4cc01c9a41c [CANDIDATE]: Term 5 pre-election: Requested pre-vote from peers e9dd218ca2244f5a934aac2b7e58eb1d (127.22.168.62:37239), c520077acf3341e29447c6c2b8d3f74c (127.22.168.61:32947)
21:04:17.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:17.771797 24086 leader_election.cc:336] T 00000000000000000000000000000000 P eb664add87f44013b596f4cc01c9a41c [CANDIDATE]: Term 5 pre-election: RPC error from VoteRequest() call to peer e9dd218ca2244f5a934aac2b7e58eb1d (127.22.168.62:37239): Network error: Client connection negotiation failed: client connection to 127.22.168.62:37239: connect: Connection refused (error 111)
21:04:17.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:17.771981 24089 leader_election.cc:336] T 00000000000000000000000000000000 P eb664add87f44013b596f4cc01c9a41c [CANDIDATE]: Term 5 pre-election: RPC error from VoteRequest() call to peer c520077acf3341e29447c6c2b8d3f74c (127.22.168.61:32947): Network error: Client connection negotiation failed: client connection to 127.22.168.61:32947: connect: Connection refused (error 111)
21:04:17.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:17.772094 24089 leader_election.cc:304] T 00000000000000000000000000000000 P eb664add87f44013b596f4cc01c9a41c [CANDIDATE]: Term 5 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: eb664add87f44013b596f4cc01c9a41c; no voters: c520077acf3341e29447c6c2b8d3f74c, e9dd218ca2244f5a934aac2b7e58eb1d
21:04:17.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:17.772363 24225 raft_consensus.cc:2749] T 00000000000000000000000000000000 P eb664add87f44013b596f4cc01c9a41c [term 4 FOLLOWER]: Leader pre-election lost for term 5. Reason: could not achieve majority
21:04:17.785 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:17.785 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:17.786 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:17.787 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:17.787 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:17.787 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:17.788 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:17.788 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:17.790 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:17.804 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:17.805 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:17.805 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:17.806 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:17.806 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:17.806 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:17.805 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:17.806 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:17.806 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:17.806 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:17.807 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:17.807 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:17.845 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:17.845 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:17.846 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:17.846 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:17.846 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:17.846 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:17.846 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:17.846 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:17.847 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:17.864 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:17.865 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:17.865 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:17.865 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:17.865 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:17.865 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:17.865 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:17.866 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:17.867 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:17.884 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:17.885 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:17.885 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:17.885 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:17.886 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:17.886 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:17.886 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:17.886 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:17.887 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:17.944 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:17.945 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:17.945 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:17.945 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:17.945 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:17.946 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:17.946 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:17.945 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:17.947 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:17.948 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=9, TimeoutTracker(timeout=250, elapsed=223), Traces: [0ms] refreshing cache from master, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [3ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [4ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [4ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [21ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [23ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [23ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [23ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [24ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [41ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [42ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [42ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [43ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [43ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [45ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [45ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [61ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [62ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [62ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [64ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [65ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [67ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [67ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [82ms] refreshing cache from master, [82ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [82ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [82ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [83ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [83ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [84ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [84ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [121ms] refreshing cache from master, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [122ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [122ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [123ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [123ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [124ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [124ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [141ms] refreshing cache from master, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [142ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [142ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [144ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [144ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [161ms] refreshing cache from master, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [162ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [162ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [163ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [163ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [163ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [164ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [220ms] refreshing cache from master, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [222ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [223ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [223ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [224ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, deferred=Deferred@629321870(state=PENDING, result=null, callback=(continuation of Deferred@1928604335 after retry RPC after error@630482639) -> (continuation of Deferred@1417996205 after retry RPC after error@1689398131) -> (continuation of Deferred@935384088 after retry RPC after error@837058097) -> (continuation of Deferred@1110330927 after retry RPC after error@926031018) -> (continuation of Deferred@115572288 after retry RPC after error@350526811) -> (continuation of Deferred@613152478 after retry RPC after error@58419625) -> (continuation of Deferred@511117048 after retry RPC after error@776556072) -> (continuation of Deferred@1427075129 after retry RPC after error@232877328), errback=(continuation of Deferred@1928604335 after retry RPC after error@630482639) -> (continuation of Deferred@1417996205 after retry RPC after error@1689398131) -> (continuation of Deferred@935384088 after retry RPC after error@837058097) -> (continuation of Deferred@1110330927 after retry RPC after error@926031018) -> (continuation of Deferred@115572288 after retry RPC after error@350526811) -> (continuation of Deferred@613152478 after retry RPC after error@58419625) -> (continuation of Deferred@511117048 after retry RPC after error@776556072) -> (continuation of Deferred@1427075129 after retry RPC after error@232877328)))
21:04:17.948 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=9, TimeoutTracker(timeout=250, elapsed=223), Traces: [0ms] refreshing cache from master, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [3ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [4ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [4ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [21ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [23ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [23ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [23ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [24ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [41ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [42ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [42ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [43ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [43ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [45ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [45ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [61ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [62ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [62ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [64ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [65ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [67ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [67ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [82ms] refreshing cache from master, [82ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [82ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [82ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [83ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [83ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [84ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [84ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [121ms] refreshing cache from master, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [122ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [122ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [123ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [123ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [124ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [124ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [141ms] refreshing cache from master, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [142ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [142ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [144ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [144ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [161ms] refreshing cache from master, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [162ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [162ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [163ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [163ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [163ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [164ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [220ms] refreshing cache from master, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [222ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [223ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [223ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [224ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, deferred=Deferred@629321870(state=PENDING, result=null, callback=(continuation of Deferred@1928604335 after retry RPC after error@630482639) -> (continuation of Deferred@1417996205 after retry RPC after error@1689398131) -> (continuation of Deferred@935384088 after retry RPC after error@837058097) -> (continuation of Deferred@1110330927 after retry RPC after error@926031018) -> (continuation of Deferred@115572288 after retry RPC after error@350526811) -> (continuation of Deferred@613152478 after retry RPC after error@58419625) -> (continuation of Deferred@511117048 after retry RPC after error@776556072) -> (continuation of Deferred@1427075129 after retry RPC after error@232877328), errback=(continuation of Deferred@1928604335 after retry RPC after error@630482639) -> (continuation of Deferred@1417996205 after retry RPC after error@1689398131) -> (continuation of Deferred@935384088 after retry RPC after error@837058097) -> (continuation of Deferred@1110330927 after retry RPC after error@926031018) -> (continuation of Deferred@115572288 after retry RPC after error@350526811) -> (continuation of Deferred@613152478 after retry RPC after error@58419625) -> (continuation of Deferred@511117048 after retry RPC after error@776556072) -> (continuation of Deferred@1427075129 after retry RPC after error@232877328)))
21:04:17.964 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:17.965 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:17.966 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:17.966 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:17.966 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:17.966 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:17.966 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:17.966 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:17.967 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:17.984 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:17.984 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:17.985 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:17.985 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:17.986 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:17.985 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:17.986 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:17.986 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:17.986 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:18.004 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:18.005 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:18.005 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:18.005 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:18.006 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:18.006 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:18.006 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:18.006 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:18.007 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:18.025 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:18.025 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:18.026 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:18.026 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:18.026 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:18.026 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:18.026 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:18.026 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:18.027 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:18.044 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:18.044 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:18.045 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:18.045 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:18.045 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:18.045 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:18.045 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:18.045 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:18.046 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:18.064 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:18.065 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:18.066 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:18.066 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:18.066 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:18.066 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:18.066 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:18.066 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:18.067 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:18.124 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:18.124 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:18.125 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:18.125 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:18.125 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:18.125 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:18.125 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:18.125 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:18.126 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:18.164 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:18.165 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:18.165 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:18.165 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:18.165 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:18.165 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:18.166 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:18.166 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:18.166 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:18.167 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=202), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [3ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [19ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [43ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [60ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [63ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [161ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [161ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [162ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [202ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [202ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [202ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, deferred=Deferred@1990573152(state=PENDING, result=null, callback=(continuation of Deferred@1649460546 after retry RPC after error@254114279) -> (continuation of Deferred@1069221116 after retry RPC after error@1680275927) -> (continuation of Deferred@3689156 after retry RPC after error@620163429) -> (continuation of Deferred@1133354467 after retry RPC after error@280378372) -> (continuation of Deferred@1385835196 after retry RPC after error@1106547795) -> (continuation of Deferred@1857645323 after retry RPC after error@1338916036) -> (continuation of Deferred@1275207755 after retry RPC after error@691102612), errback=(continuation of Deferred@1649460546 after retry RPC after error@254114279) -> (continuation of Deferred@1069221116 after retry RPC after error@1680275927) -> (continuation of Deferred@3689156 after retry RPC after error@620163429) -> (continuation of Deferred@1133354467 after retry RPC after error@280378372) -> (continuation of Deferred@1385835196 after retry RPC after error@1106547795) -> (continuation of Deferred@1857645323 after retry RPC after error@1338916036) -> (continuation of Deferred@1275207755 after retry RPC after error@691102612)))
21:04:18.167 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=202), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [3ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [19ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [43ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [60ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [63ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [161ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [161ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [162ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [202ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [202ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [202ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, deferred=Deferred@1990573152(state=PENDING, result=null, callback=(continuation of Deferred@1649460546 after retry RPC after error@254114279) -> (continuation of Deferred@1069221116 after retry RPC after error@1680275927) -> (continuation of Deferred@3689156 after retry RPC after error@620163429) -> (continuation of Deferred@1133354467 after retry RPC after error@280378372) -> (continuation of Deferred@1385835196 after retry RPC after error@1106547795) -> (continuation of Deferred@1857645323 after retry RPC after error@1338916036) -> (continuation of Deferred@1275207755 after retry RPC after error@691102612), errback=(continuation of Deferred@1649460546 after retry RPC after error@254114279) -> (continuation of Deferred@1069221116 after retry RPC after error@1680275927) -> (continuation of Deferred@3689156 after retry RPC after error@620163429) -> (continuation of Deferred@1133354467 after retry RPC after error@280378372) -> (continuation of Deferred@1385835196 after retry RPC after error@1106547795) -> (continuation of Deferred@1857645323 after retry RPC after error@1338916036) -> (continuation of Deferred@1275207755 after retry RPC after error@691102612)))
21:04:18.184 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:18.185 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:18.186 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:18.186 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:18.186 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:18.186 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:18.186 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:18.186 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:18.187 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:18.205 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:18.205 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:18.206 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:18.206 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:18.206 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:18.206 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:18.206 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:18.206 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:18.207 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:18.224 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:18.225 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:18.225 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:18.225 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:18.225 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:18.225 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:18.226 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:18.226 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:18.226 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:18.244 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:18.245 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:18.245 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:18.245 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:18.246 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:18.246 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:18.246 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:18.246 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:18.246 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:18.264 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:18.265 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:18.266 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:18.266 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:18.266 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:18.266 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:18.266 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:18.266 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:18.267 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:18.304 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:18.305 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:18.305 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:18.305 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:18.305 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:18.305 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:18.306 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:18.306 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:18.306 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:18.344 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:18.345 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:18.346 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:18.346 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:18.346 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:18.346 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:18.346 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:18.346 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:18.346 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:18.347 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=162), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [23ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [83ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [162ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [162ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [162ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [163ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, deferred=Deferred@1516291696(state=PENDING, result=null, callback=(continuation of Deferred@931986098 after retry RPC after error@1944932678) -> (continuation of Deferred@1047605944 after retry RPC after error@1652432960) -> (continuation of Deferred@855056343 after retry RPC after error@722014126) -> (continuation of Deferred@475667589 after retry RPC after error@263541614) -> (continuation of Deferred@95878376 after retry RPC after error@1721890703) -> (continuation of Deferred@710111722 after retry RPC after error@265845229), errback=(continuation of Deferred@931986098 after retry RPC after error@1944932678) -> (continuation of Deferred@1047605944 after retry RPC after error@1652432960) -> (continuation of Deferred@855056343 after retry RPC after error@722014126) -> (continuation of Deferred@475667589 after retry RPC after error@263541614) -> (continuation of Deferred@95878376 after retry RPC after error@1721890703) -> (continuation of Deferred@710111722 after retry RPC after error@265845229)))
21:04:18.347 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=162), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [23ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [83ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [162ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [162ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [162ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [163ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, deferred=Deferred@1516291696(state=PENDING, result=null, callback=(continuation of Deferred@931986098 after retry RPC after error@1944932678) -> (continuation of Deferred@1047605944 after retry RPC after error@1652432960) -> (continuation of Deferred@855056343 after retry RPC after error@722014126) -> (continuation of Deferred@475667589 after retry RPC after error@263541614) -> (continuation of Deferred@95878376 after retry RPC after error@1721890703) -> (continuation of Deferred@710111722 after retry RPC after error@265845229), errback=(continuation of Deferred@931986098 after retry RPC after error@1944932678) -> (continuation of Deferred@1047605944 after retry RPC after error@1652432960) -> (continuation of Deferred@855056343 after retry RPC after error@722014126) -> (continuation of Deferred@475667589 after retry RPC after error@263541614) -> (continuation of Deferred@95878376 after retry RPC after error@1721890703) -> (continuation of Deferred@710111722 after retry RPC after error@265845229)))
21:04:18.365 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:18.365 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:18.366 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:18.366 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:18.366 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:18.366 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:18.366 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:18.366 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:18.368 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:18.385 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:18.385 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:18.386 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:18.386 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:18.386 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:18.386 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:18.386 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:18.386 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:18.387 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:18.404 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:18.404 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:18.405 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:18.405 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:18.405 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:18.405 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:18.405 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:18.405 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:18.406 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:18.424 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:18.425 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:18.426 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:18.426 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:18.426 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:18.426 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:18.426 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:18.426 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:18.427 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:18.445 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:18.445 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:18.446 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:18.446 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:18.446 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:18.446 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:18.446 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:18.446 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:18.447 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:18.464 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:18.465 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:18.466 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:18.466 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:18.466 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:18.466 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:18.466 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:18.466 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:18.467 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:18.524 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:18.525 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:18.525 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:18.525 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:18.525 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:18.526 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:18.526 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:18.526 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:18.527 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:18.527 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=162), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [4ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [4ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [20ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [23ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [39ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [83ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [103ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [104ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [161ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [162ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [162ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [163ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, deferred=Deferred@1114839786(state=PENDING, result=null, callback=(continuation of Deferred@167273289 after retry RPC after error@1639061792) -> (continuation of Deferred@1024468678 after retry RPC after error@1735319492) -> (continuation of Deferred@759568971 after retry RPC after error@1556901338) -> (continuation of Deferred@1624659603 after retry RPC after error@1901540786) -> (continuation of Deferred@1818160529 after retry RPC after error@1405349757) -> (continuation of Deferred@24336728 after retry RPC after error@1228998241), errback=(continuation of Deferred@167273289 after retry RPC after error@1639061792) -> (continuation of Deferred@1024468678 after retry RPC after error@1735319492) -> (continuation of Deferred@759568971 after retry RPC after error@1556901338) -> (continuation of Deferred@1624659603 after retry RPC after error@1901540786) -> (continuation of Deferred@1818160529 after retry RPC after error@1405349757) -> (continuation of Deferred@24336728 after retry RPC after error@1228998241)))
21:04:18.528 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=162), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [4ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [4ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [20ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [23ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [39ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [83ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [103ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [104ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [161ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [162ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [162ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [163ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, deferred=Deferred@1114839786(state=PENDING, result=null, callback=(continuation of Deferred@167273289 after retry RPC after error@1639061792) -> (continuation of Deferred@1024468678 after retry RPC after error@1735319492) -> (continuation of Deferred@759568971 after retry RPC after error@1556901338) -> (continuation of Deferred@1624659603 after retry RPC after error@1901540786) -> (continuation of Deferred@1818160529 after retry RPC after error@1405349757) -> (continuation of Deferred@24336728 after retry RPC after error@1228998241), errback=(continuation of Deferred@167273289 after retry RPC after error@1639061792) -> (continuation of Deferred@1024468678 after retry RPC after error@1735319492) -> (continuation of Deferred@759568971 after retry RPC after error@1556901338) -> (continuation of Deferred@1624659603 after retry RPC after error@1901540786) -> (continuation of Deferred@1818160529 after retry RPC after error@1405349757) -> (continuation of Deferred@24336728 after retry RPC after error@1228998241)))
21:04:18.544 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:18.545 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:18.545 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:18.545 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:18.545 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:18.546 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:18.546 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:18.546 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:18.547 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:18.564 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:18.565 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:18.566 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:18.566 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:18.566 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:18.567 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:18.567 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:18.567 [INFO - pool-40-thread-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:18.567 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:18.584 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:18.585 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:18.585 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:18.586 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:18.586 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:18.586 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:18.586 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:18.587 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:18.587 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:18.604 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:18.605 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:18.605 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:18.605 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:18.605 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:18.605 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:18.606 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:18.606 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:18.607 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:18.624 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:18.625 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:18.625 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:18.625 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:18.626 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:18.626 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:18.626 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:18.626 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:18.627 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:18.644 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:18.645 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:18.645 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:18.645 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:18.645 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:18.646 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:18.646 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:18.646 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:18.647 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:18.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:18.691751 24232 raft_consensus.cc:493] T 00000000000000000000000000000000 P eb664add87f44013b596f4cc01c9a41c [term 4 FOLLOWER]: Starting pre-election (detected failure of leader c520077acf3341e29447c6c2b8d3f74c)
21:04:18.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:18.691946 24232 raft_consensus.cc:515] T 00000000000000000000000000000000 P eb664add87f44013b596f4cc01c9a41c [term 4 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e9dd218ca2244f5a934aac2b7e58eb1d" member_type: VOTER last_known_addr { host: "127.22.168.62" port: 37239 } } peers { permanent_uuid: "c520077acf3341e29447c6c2b8d3f74c" member_type: VOTER last_known_addr { host: "127.22.168.61" port: 32947 } } peers { permanent_uuid: "eb664add87f44013b596f4cc01c9a41c" member_type: VOTER last_known_addr { host: "127.22.168.60" port: 33247 } }
21:04:18.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:18.692662 24232 leader_election.cc:290] T 00000000000000000000000000000000 P eb664add87f44013b596f4cc01c9a41c [CANDIDATE]: Term 5 pre-election: Requested pre-vote from peers e9dd218ca2244f5a934aac2b7e58eb1d (127.22.168.62:37239), c520077acf3341e29447c6c2b8d3f74c (127.22.168.61:32947)
21:04:18.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:18.694576 24086 leader_election.cc:336] T 00000000000000000000000000000000 P eb664add87f44013b596f4cc01c9a41c [CANDIDATE]: Term 5 pre-election: RPC error from VoteRequest() call to peer e9dd218ca2244f5a934aac2b7e58eb1d (127.22.168.62:37239): Network error: Client connection negotiation failed: client connection to 127.22.168.62:37239: connect: Connection refused (error 111)
21:04:18.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:18.694746 24089 leader_election.cc:336] T 00000000000000000000000000000000 P eb664add87f44013b596f4cc01c9a41c [CANDIDATE]: Term 5 pre-election: RPC error from VoteRequest() call to peer c520077acf3341e29447c6c2b8d3f74c (127.22.168.61:32947): Network error: Client connection negotiation failed: client connection to 127.22.168.61:32947: connect: Connection refused (error 111)
21:04:18.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:18.694849 24089 leader_election.cc:304] T 00000000000000000000000000000000 P eb664add87f44013b596f4cc01c9a41c [CANDIDATE]: Term 5 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: eb664add87f44013b596f4cc01c9a41c; no voters: c520077acf3341e29447c6c2b8d3f74c, e9dd218ca2244f5a934aac2b7e58eb1d
21:04:18.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:18.695067 24232 raft_consensus.cc:2749] T 00000000000000000000000000000000 P eb664add87f44013b596f4cc01c9a41c [term 4 FOLLOWER]: Leader pre-election lost for term 5. Reason: could not achieve majority
21:04:18.705 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:18.705 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:18.706 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:18.706 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:18.706 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:18.706 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:18.706 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:18.706 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:18.707 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:18.707 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=162), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [23ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [23ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [43ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [43ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [103ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [162ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [162ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [163ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [163ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, deferred=Deferred@1169361250(state=PENDING, result=null, callback=(continuation of Deferred@1274132981 after retry RPC after error@340270) -> (continuation of Deferred@2043193235 after retry RPC after error@611146193) -> (continuation of Deferred@1645568696 after retry RPC after error@1261482901) -> (continuation of Deferred@2004382047 after retry RPC after error@957685168) -> (continuation of Deferred@1885124744 after retry RPC after error@1044250893) -> (continuation of Deferred@184053020 after retry RPC after error@462494062), errback=(continuation of Deferred@1274132981 after retry RPC after error@340270) -> (continuation of Deferred@2043193235 after retry RPC after error@611146193) -> (continuation of Deferred@1645568696 after retry RPC after error@1261482901) -> (continuation of Deferred@2004382047 after retry RPC after error@957685168) -> (continuation of Deferred@1885124744 after retry RPC after error@1044250893) -> (continuation of Deferred@184053020 after retry RPC after error@462494062)))
21:04:18.708 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=162), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [23ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [23ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [43ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [43ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [103ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [162ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [162ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [163ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [163ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, deferred=Deferred@1169361250(state=PENDING, result=null, callback=(continuation of Deferred@1274132981 after retry RPC after error@340270) -> (continuation of Deferred@2043193235 after retry RPC after error@611146193) -> (continuation of Deferred@1645568696 after retry RPC after error@1261482901) -> (continuation of Deferred@2004382047 after retry RPC after error@957685168) -> (continuation of Deferred@1885124744 after retry RPC after error@1044250893) -> (continuation of Deferred@184053020 after retry RPC after error@462494062), errback=(continuation of Deferred@1274132981 after retry RPC after error@340270) -> (continuation of Deferred@2043193235 after retry RPC after error@611146193) -> (continuation of Deferred@1645568696 after retry RPC after error@1261482901) -> (continuation of Deferred@2004382047 after retry RPC after error@957685168) -> (continuation of Deferred@1885124744 after retry RPC after error@1044250893) -> (continuation of Deferred@184053020 after retry RPC after error@462494062)))
21:04:18.725 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:18.725 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:18.726 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:18.726 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:18.726 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:18.727 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:18.727 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:18.727 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:18.727 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:18.744 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:18.744 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:18.745 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:18.745 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:18.745 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:18.745 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:18.745 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:18.745 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:18.746 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:18.764 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:18.764 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:18.765 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:18.765 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:18.765 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:18.765 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:18.765 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:18.765 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:18.766 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:18.784 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:18.785 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:18.785 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:18.785 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:18.785 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:18.785 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:18.786 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:18.786 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:18.786 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:18.804 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:18.805 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:18.805 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:18.805 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:18.805 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:18.805 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:18.806 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:18.806 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:18.806 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:18.844 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:18.845 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:18.845 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:18.845 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:18.846 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:18.846 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:18.846 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:18.846 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:18.847 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:18.904 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:18.904 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:18.905 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:18.905 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:18.905 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:18.905 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:18.905 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:18.905 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:18.906 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:18.964 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:18.965 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:18.965 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:18.966 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:18.966 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:18.966 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:18.966 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:18.966 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:18.967 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:18.968 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=242), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [3ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [3ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [3ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [123ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [179ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [181ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [181ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [182ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [242ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [242ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [242ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [243ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, deferred=Deferred@1651909072(state=PENDING, result=null, callback=(continuation of Deferred@937490259 after retry RPC after error@1876619078) -> (continuation of Deferred@927712206 after retry RPC after error@1692404822) -> (continuation of Deferred@1409955318 after retry RPC after error@1813139694) -> (continuation of Deferred@95337233 after retry RPC after error@1632146766) -> (continuation of Deferred@1684580628 after retry RPC after error@436408407) -> (continuation of Deferred@1787119157 after retry RPC after error@1338133710) -> (continuation of Deferred@1858305067 after retry RPC after error@1824540684), errback=(continuation of Deferred@937490259 after retry RPC after error@1876619078) -> (continuation of Deferred@927712206 after retry RPC after error@1692404822) -> (continuation of Deferred@1409955318 after retry RPC after error@1813139694) -> (continuation of Deferred@95337233 after retry RPC after error@1632146766) -> (continuation of Deferred@1684580628 after retry RPC after error@436408407) -> (continuation of Deferred@1787119157 after retry RPC after error@1338133710) -> (continuation of Deferred@1858305067 after retry RPC after error@1824540684)))
21:04:18.968 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=242), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [3ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [3ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [3ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [123ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [179ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [181ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [181ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [182ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [242ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [242ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [242ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [243ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, deferred=Deferred@1651909072(state=PENDING, result=null, callback=(continuation of Deferred@937490259 after retry RPC after error@1876619078) -> (continuation of Deferred@927712206 after retry RPC after error@1692404822) -> (continuation of Deferred@1409955318 after retry RPC after error@1813139694) -> (continuation of Deferred@95337233 after retry RPC after error@1632146766) -> (continuation of Deferred@1684580628 after retry RPC after error@436408407) -> (continuation of Deferred@1787119157 after retry RPC after error@1338133710) -> (continuation of Deferred@1858305067 after retry RPC after error@1824540684), errback=(continuation of Deferred@937490259 after retry RPC after error@1876619078) -> (continuation of Deferred@927712206 after retry RPC after error@1692404822) -> (continuation of Deferred@1409955318 after retry RPC after error@1813139694) -> (continuation of Deferred@95337233 after retry RPC after error@1632146766) -> (continuation of Deferred@1684580628 after retry RPC after error@436408407) -> (continuation of Deferred@1787119157 after retry RPC after error@1338133710) -> (continuation of Deferred@1858305067 after retry RPC after error@1824540684)))
21:04:18.985 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:18.985 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:18.986 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:18.986 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:18.986 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:18.986 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:18.986 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:18.986 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:18.987 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:19.004 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:19.004 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:19.005 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:19.005 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:19.005 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:19.006 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:19.006 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:19.006 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:19.007 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:19.024 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:19.025 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:19.025 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:19.025 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:19.025 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:19.025 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:19.025 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:19.025 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:19.027 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:19.044 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:19.045 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:19.045 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:19.045 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:19.046 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:19.046 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:19.046 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:19.046 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:19.047 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:19.064 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:19.065 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:19.066 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:19.066 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:19.066 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:19.066 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:19.066 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:19.066 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:19.067 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:19.104 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:19.105 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:19.105 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:19.105 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:19.105 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:19.105 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:19.105 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:19.106 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:19.106 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:19.164 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:19.165 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:19.166 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:19.166 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:19.166 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:19.166 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:19.166 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:19.166 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:19.167 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:19.205 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:19.205 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:19.206 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:19.206 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:19.206 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:19.206 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:19.206 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:19.206 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:19.207 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:19.208 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=223), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [3ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [19ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [23ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [83ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [119ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [182ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [182ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [183ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [183ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [220ms] refreshing cache from master, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [222ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [222ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [223ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [223ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, deferred=Deferred@975698805(state=PENDING, result=null, callback=(continuation of Deferred@1989995072 after retry RPC after error@181364209) -> (continuation of Deferred@1970563122 after retry RPC after error@1879955898) -> (continuation of Deferred@1632826129 after retry RPC after error@281272871) -> (continuation of Deferred@1184906741 after retry RPC after error@1484713106) -> (continuation of Deferred@784484087 after retry RPC after error@564091858) -> (continuation of Deferred@1712817537 after retry RPC after error@1227030445) -> (continuation of Deferred@1970007424 after retry RPC after error@787573044), errback=(continuation of Deferred@1989995072 after retry RPC after error@181364209) -> (continuation of Deferred@1970563122 after retry RPC after error@1879955898) -> (continuation of Deferred@1632826129 after retry RPC after error@281272871) -> (continuation of Deferred@1184906741 after retry RPC after error@1484713106) -> (continuation of Deferred@784484087 after retry RPC after error@564091858) -> (continuation of Deferred@1712817537 after retry RPC after error@1227030445) -> (continuation of Deferred@1970007424 after retry RPC after error@787573044)))
21:04:19.208 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=223), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [3ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [19ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [23ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [83ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [119ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [182ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [182ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [183ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [183ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [220ms] refreshing cache from master, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [222ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [222ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [223ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [223ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, deferred=Deferred@975698805(state=PENDING, result=null, callback=(continuation of Deferred@1989995072 after retry RPC after error@181364209) -> (continuation of Deferred@1970563122 after retry RPC after error@1879955898) -> (continuation of Deferred@1632826129 after retry RPC after error@281272871) -> (continuation of Deferred@1184906741 after retry RPC after error@1484713106) -> (continuation of Deferred@784484087 after retry RPC after error@564091858) -> (continuation of Deferred@1712817537 after retry RPC after error@1227030445) -> (continuation of Deferred@1970007424 after retry RPC after error@787573044), errback=(continuation of Deferred@1989995072 after retry RPC after error@181364209) -> (continuation of Deferred@1970563122 after retry RPC after error@1879955898) -> (continuation of Deferred@1632826129 after retry RPC after error@281272871) -> (continuation of Deferred@1184906741 after retry RPC after error@1484713106) -> (continuation of Deferred@784484087 after retry RPC after error@564091858) -> (continuation of Deferred@1712817537 after retry RPC after error@1227030445) -> (continuation of Deferred@1970007424 after retry RPC after error@787573044)))
21:04:19.224 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:19.224 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:19.225 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:19.225 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:19.225 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:19.226 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:19.226 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:19.226 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:19.226 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:19.244 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:19.245 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:19.245 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:19.245 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:19.245 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:19.245 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:19.245 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:19.245 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:19.246 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:19.264 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:19.265 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:19.265 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:19.266 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:19.266 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:19.266 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:19.266 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:19.266 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:19.267 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:19.284 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:19.285 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:19.285 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:19.285 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:19.285 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:19.286 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:19.286 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:19.286 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:19.287 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:19.304 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:19.305 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:19.305 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:19.306 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:19.306 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:19.306 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:19.306 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:19.306 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:19.306 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:19.324 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:19.325 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:19.326 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:19.326 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:19.326 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:19.326 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:19.326 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:19.326 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:19.327 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:19.384 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:19.385 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:19.385 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:19.385 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:19.385 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:19.385 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:19.386 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:19.386 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:19.386 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:19.465 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:19.465 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:19.466 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:19.466 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:19.466 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:19.466 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:19.466 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:19.466 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:19.467 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:19.467 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=243), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [3ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [43ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [103ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [161ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [161ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [162ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [163ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [242ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [242ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [243ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [243ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, deferred=Deferred@2115953654(state=PENDING, result=null, callback=(continuation of Deferred@1930263191 after retry RPC after error@404994883) -> (continuation of Deferred@699763174 after retry RPC after error@549314165) -> (continuation of Deferred@947700023 after retry RPC after error@1031838936) -> (continuation of Deferred@956291127 after retry RPC after error@1988097835) -> (continuation of Deferred@685045162 after retry RPC after error@294378036) -> (continuation of Deferred@1475581522 after retry RPC after error@1589437736) -> (continuation of Deferred@1372205808 after retry RPC after error@509548679), errback=(continuation of Deferred@1930263191 after retry RPC after error@404994883) -> (continuation of Deferred@699763174 after retry RPC after error@549314165) -> (continuation of Deferred@947700023 after retry RPC after error@1031838936) -> (continuation of Deferred@956291127 after retry RPC after error@1988097835) -> (continuation of Deferred@685045162 after retry RPC after error@294378036) -> (continuation of Deferred@1475581522 after retry RPC after error@1589437736) -> (continuation of Deferred@1372205808 after retry RPC after error@509548679)))
21:04:19.468 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=243), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [3ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [43ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [103ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [161ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [161ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [162ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [163ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [242ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [242ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [243ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [243ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, deferred=Deferred@2115953654(state=PENDING, result=null, callback=(continuation of Deferred@1930263191 after retry RPC after error@404994883) -> (continuation of Deferred@699763174 after retry RPC after error@549314165) -> (continuation of Deferred@947700023 after retry RPC after error@1031838936) -> (continuation of Deferred@956291127 after retry RPC after error@1988097835) -> (continuation of Deferred@685045162 after retry RPC after error@294378036) -> (continuation of Deferred@1475581522 after retry RPC after error@1589437736) -> (continuation of Deferred@1372205808 after retry RPC after error@509548679), errback=(continuation of Deferred@1930263191 after retry RPC after error@404994883) -> (continuation of Deferred@699763174 after retry RPC after error@549314165) -> (continuation of Deferred@947700023 after retry RPC after error@1031838936) -> (continuation of Deferred@956291127 after retry RPC after error@1988097835) -> (continuation of Deferred@685045162 after retry RPC after error@294378036) -> (continuation of Deferred@1475581522 after retry RPC after error@1589437736) -> (continuation of Deferred@1372205808 after retry RPC after error@509548679)))
21:04:19.485 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:19.485 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:19.486 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:19.486 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:19.486 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:19.486 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:19.486 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:19.486 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:19.487 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:19.505 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:19.505 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:19.506 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:19.506 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:19.506 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:19.506 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:19.506 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:19.506 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:19.507 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:19.524 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:19.524 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:19.525 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:19.525 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:19.525 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:19.525 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:19.526 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:19.526 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:19.526 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:19.544 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:19.545 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:19.545 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:19.545 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:19.545 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:19.545 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:19.546 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:19.546 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:19.546 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:19.564 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:19.565 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:19.565 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:19.565 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:19.566 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:19.566 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:19.565 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:19.566 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:19.566 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:19.566 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:19.567 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:19.567 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:19.584 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:19.585 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:19.585 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:19.585 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:19.585 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:19.585 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:19.586 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:19.586 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:19.586 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:19.645 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:19.645 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:19.646 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:19.646 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:19.646 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:19.646 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:19.646 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:19.646 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:19.647 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:19.684 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:19.684 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:19.685 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:19.685 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:19.685 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:19.685 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:19.685 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:19.685 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:19.686 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:19.687 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=201), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [20ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [23ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [81ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [83ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [160ms] refreshing cache from master, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [162ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [162ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [163ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [163ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [201ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [201ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [202ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, deferred=Deferred@1776364375(state=PENDING, result=null, callback=(continuation of Deferred@1707492263 after retry RPC after error@472570493) -> (continuation of Deferred@1286005498 after retry RPC after error@754832298) -> (continuation of Deferred@2079422185 after retry RPC after error@826450648) -> (continuation of Deferred@1242052408 after retry RPC after error@145415504) -> (continuation of Deferred@2023535762 after retry RPC after error@1550622011) -> (continuation of Deferred@1473695865 after retry RPC after error@155764622) -> (continuation of Deferred@342739473 after retry RPC after error@751835362), errback=(continuation of Deferred@1707492263 after retry RPC after error@472570493) -> (continuation of Deferred@1286005498 after retry RPC after error@754832298) -> (continuation of Deferred@2079422185 after retry RPC after error@826450648) -> (continuation of Deferred@1242052408 after retry RPC after error@145415504) -> (continuation of Deferred@2023535762 after retry RPC after error@1550622011) -> (continuation of Deferred@1473695865 after retry RPC after error@155764622) -> (continuation of Deferred@342739473 after retry RPC after error@751835362)))
21:04:19.687 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=201), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [20ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [23ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [81ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [83ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [160ms] refreshing cache from master, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [162ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [162ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [163ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [163ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [201ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [201ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [202ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, deferred=Deferred@1776364375(state=PENDING, result=null, callback=(continuation of Deferred@1707492263 after retry RPC after error@472570493) -> (continuation of Deferred@1286005498 after retry RPC after error@754832298) -> (continuation of Deferred@2079422185 after retry RPC after error@826450648) -> (continuation of Deferred@1242052408 after retry RPC after error@145415504) -> (continuation of Deferred@2023535762 after retry RPC after error@1550622011) -> (continuation of Deferred@1473695865 after retry RPC after error@155764622) -> (continuation of Deferred@342739473 after retry RPC after error@751835362), errback=(continuation of Deferred@1707492263 after retry RPC after error@472570493) -> (continuation of Deferred@1286005498 after retry RPC after error@754832298) -> (continuation of Deferred@2079422185 after retry RPC after error@826450648) -> (continuation of Deferred@1242052408 after retry RPC after error@145415504) -> (continuation of Deferred@2023535762 after retry RPC after error@1550622011) -> (continuation of Deferred@1473695865 after retry RPC after error@155764622) -> (continuation of Deferred@342739473 after retry RPC after error@751835362)))
21:04:19.704 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:19.704 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:19.705 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:19.705 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:19.705 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:19.705 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:19.705 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:19.705 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:19.706 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:19.724 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:19.725 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:19.725 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:19.725 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:19.725 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:19.726 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:19.725 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:19.726 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:19.726 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:19.744 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:19.745 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:19.745 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:19.745 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:19.745 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:19.745 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:19.746 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:19.746 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:19.747 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:19.764 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:19.765 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:19.765 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:19.766 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:19.766 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:19.766 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:19.766 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:19.766 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:19.767 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:19.785 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:19.785 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:19.786 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:19.786 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:19.786 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:19.786 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:19.787 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:19.787 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:19.787 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:19.824 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:19.825 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:19.826 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:19.826 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:19.826 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:19.826 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:19.826 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:19.826 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:19.829 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:19.884 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:19.885 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:19.885 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:19.885 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:19.886 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:19.886 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:19.886 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:19.886 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:19.886 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:19.887 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=182), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [63ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [83ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [83ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [125ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [125ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [181ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [182ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [182ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, deferred=Deferred@968817174(state=PENDING, result=null, callback=(continuation of Deferred@225896006 after retry RPC after error@1970969163) -> (continuation of Deferred@239700265 after retry RPC after error@1080667324) -> (continuation of Deferred@2048341537 after retry RPC after error@1528714218) -> (continuation of Deferred@425494881 after retry RPC after error@341695162) -> (continuation of Deferred@1463308334 after retry RPC after error@2123421728) -> (continuation of Deferred@66732306 after retry RPC after error@2030617565), errback=(continuation of Deferred@225896006 after retry RPC after error@1970969163) -> (continuation of Deferred@239700265 after retry RPC after error@1080667324) -> (continuation of Deferred@2048341537 after retry RPC after error@1528714218) -> (continuation of Deferred@425494881 after retry RPC after error@341695162) -> (continuation of Deferred@1463308334 after retry RPC after error@2123421728) -> (continuation of Deferred@66732306 after retry RPC after error@2030617565)))
21:04:19.887 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=182), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [63ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [83ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [83ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [125ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [125ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [181ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [182ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [182ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, deferred=Deferred@968817174(state=PENDING, result=null, callback=(continuation of Deferred@225896006 after retry RPC after error@1970969163) -> (continuation of Deferred@239700265 after retry RPC after error@1080667324) -> (continuation of Deferred@2048341537 after retry RPC after error@1528714218) -> (continuation of Deferred@425494881 after retry RPC after error@341695162) -> (continuation of Deferred@1463308334 after retry RPC after error@2123421728) -> (continuation of Deferred@66732306 after retry RPC after error@2030617565), errback=(continuation of Deferred@225896006 after retry RPC after error@1970969163) -> (continuation of Deferred@239700265 after retry RPC after error@1080667324) -> (continuation of Deferred@2048341537 after retry RPC after error@1528714218) -> (continuation of Deferred@425494881 after retry RPC after error@341695162) -> (continuation of Deferred@1463308334 after retry RPC after error@2123421728) -> (continuation of Deferred@66732306 after retry RPC after error@2030617565)))
21:04:19.904 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:19.905 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:19.905 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:19.905 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:19.905 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:19.905 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:19.906 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:19.906 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:19.906 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:19.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:19.911536 24071 master_runner.cc:186] Error getting master registration for 127.22.168.62:37239: Network error: Client connection negotiation failed: client connection to 127.22.168.62:37239: connect: Connection refused (error 111),
21:04:19.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:19.911729 24071 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
21:04:19.924 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:19.925 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:19.925 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:19.926 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:19.926 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:19.926 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:19.926 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:19.926 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:19.927 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:19.944 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:19.945 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:19.945 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:19.945 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:19.945 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:19.946 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:19.946 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:19.946 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:19.946 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:19.964 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:19.965 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:19.965 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:19.965 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:19.965 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:19.965 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:19.966 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:19.966 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:19.966 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:19.984 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:19.985 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:19.985 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:19.985 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:19.985 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:19.986 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:19.986 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:19.986 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:19.987 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:20.004 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:20.005 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:20.006 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:20.006 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:20.006 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:20.006 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:20.006 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:20.006 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:20.006 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:20.044 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:20.044 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:20.045 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:20.045 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:20.045 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:20.045 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:20.045 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:20.045 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:20.046 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:20.124 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:20.125 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:20.125 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:20.125 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:20.126 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:20.126 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:20.126 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:20.126 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:20.127 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:20.127 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=222), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [23ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [139ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [141ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [141ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [142ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [222ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [222ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [223ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [223ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, deferred=Deferred@2086249363(state=PENDING, result=null, callback=(continuation of Deferred@640487203 after retry RPC after error@1024249171) -> (continuation of Deferred@1046636565 after retry RPC after error@253100401) -> (continuation of Deferred@1091326887 after retry RPC after error@447838336) -> (continuation of Deferred@926849283 after retry RPC after error@136059676) -> (continuation of Deferred@724496595 after retry RPC after error@986380366) -> (continuation of Deferred@2038405269 after retry RPC after error@299087965) -> (continuation of Deferred@1294428892 after retry RPC after error@549647294), errback=(continuation of Deferred@640487203 after retry RPC after error@1024249171) -> (continuation of Deferred@1046636565 after retry RPC after error@253100401) -> (continuation of Deferred@1091326887 after retry RPC after error@447838336) -> (continuation of Deferred@926849283 after retry RPC after error@136059676) -> (continuation of Deferred@724496595 after retry RPC after error@986380366) -> (continuation of Deferred@2038405269 after retry RPC after error@299087965) -> (continuation of Deferred@1294428892 after retry RPC after error@549647294)))
21:04:20.128 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=222), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [23ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [139ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [141ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [141ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [142ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [222ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [222ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [223ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [223ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, deferred=Deferred@2086249363(state=PENDING, result=null, callback=(continuation of Deferred@640487203 after retry RPC after error@1024249171) -> (continuation of Deferred@1046636565 after retry RPC after error@253100401) -> (continuation of Deferred@1091326887 after retry RPC after error@447838336) -> (continuation of Deferred@926849283 after retry RPC after error@136059676) -> (continuation of Deferred@724496595 after retry RPC after error@986380366) -> (continuation of Deferred@2038405269 after retry RPC after error@299087965) -> (continuation of Deferred@1294428892 after retry RPC after error@549647294), errback=(continuation of Deferred@640487203 after retry RPC after error@1024249171) -> (continuation of Deferred@1046636565 after retry RPC after error@253100401) -> (continuation of Deferred@1091326887 after retry RPC after error@447838336) -> (continuation of Deferred@926849283 after retry RPC after error@136059676) -> (continuation of Deferred@724496595 after retry RPC after error@986380366) -> (continuation of Deferred@2038405269 after retry RPC after error@299087965) -> (continuation of Deferred@1294428892 after retry RPC after error@549647294)))
21:04:20.144 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:20.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:20.145153 24245 raft_consensus.cc:493] T 00000000000000000000000000000000 P eb664add87f44013b596f4cc01c9a41c [term 4 FOLLOWER]: Starting pre-election (detected failure of leader c520077acf3341e29447c6c2b8d3f74c)
21:04:20.145 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:20.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:20.145365 24245 raft_consensus.cc:515] T 00000000000000000000000000000000 P eb664add87f44013b596f4cc01c9a41c [term 4 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e9dd218ca2244f5a934aac2b7e58eb1d" member_type: VOTER last_known_addr { host: "127.22.168.62" port: 37239 } } peers { permanent_uuid: "c520077acf3341e29447c6c2b8d3f74c" member_type: VOTER last_known_addr { host: "127.22.168.61" port: 32947 } } peers { permanent_uuid: "eb664add87f44013b596f4cc01c9a41c" member_type: VOTER last_known_addr { host: "127.22.168.60" port: 33247 } }
21:04:20.145 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:20.146 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:20.146 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:20.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:20.146481 24245 leader_election.cc:290] T 00000000000000000000000000000000 P eb664add87f44013b596f4cc01c9a41c [CANDIDATE]: Term 5 pre-election: Requested pre-vote from peers e9dd218ca2244f5a934aac2b7e58eb1d (127.22.168.62:37239), c520077acf3341e29447c6c2b8d3f74c (127.22.168.61:32947)
21:04:20.146 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:20.146 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:20.146 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:20.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:20.147032 24089 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.22.168.61:32947: connect: Connection refused (error 111) [suppressed 8 similar messages]
21:04:20.147 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:20.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:20.148538 24086 leader_election.cc:336] T 00000000000000000000000000000000 P eb664add87f44013b596f4cc01c9a41c [CANDIDATE]: Term 5 pre-election: RPC error from VoteRequest() call to peer e9dd218ca2244f5a934aac2b7e58eb1d (127.22.168.62:37239): Network error: Client connection negotiation failed: client connection to 127.22.168.62:37239: connect: Connection refused (error 111)
21:04:20.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:20.149041 24089 leader_election.cc:336] T 00000000000000000000000000000000 P eb664add87f44013b596f4cc01c9a41c [CANDIDATE]: Term 5 pre-election: RPC error from VoteRequest() call to peer c520077acf3341e29447c6c2b8d3f74c (127.22.168.61:32947): Network error: Client connection negotiation failed: client connection to 127.22.168.61:32947: connect: Connection refused (error 111)
21:04:20.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:20.149181 24089 leader_election.cc:304] T 00000000000000000000000000000000 P eb664add87f44013b596f4cc01c9a41c [CANDIDATE]: Term 5 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: eb664add87f44013b596f4cc01c9a41c; no voters: c520077acf3341e29447c6c2b8d3f74c, e9dd218ca2244f5a934aac2b7e58eb1d
21:04:20.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:20.149514 24245 raft_consensus.cc:2749] T 00000000000000000000000000000000 P eb664add87f44013b596f4cc01c9a41c [term 4 FOLLOWER]: Leader pre-election lost for term 5. Reason: could not achieve majority
21:04:20.165 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:20.165 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:20.166 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:20.166 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:20.166 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:20.166 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:20.166 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:20.166 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:20.167 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:20.184 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:20.184 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:20.185 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:20.185 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:20.185 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:20.185 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:20.185 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:20.185 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:20.186 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:20.204 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:20.204 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:20.205 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:20.205 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:20.205 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:20.205 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:20.205 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:20.205 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:20.206 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:20.224 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:20.225 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:20.225 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:20.225 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:20.225 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:20.225 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:20.225 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:20.226 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:20.226 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:20.265 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:20.265 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:20.266 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:20.266 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:20.266 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:20.266 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:20.266 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:20.266 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:20.267 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:20.284 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:20.284 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:20.285 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:20.285 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:20.285 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:20.285 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:20.285 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:20.285 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:20.286 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:20.344 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:20.345 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:20.345 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:20.345 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:20.345 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:20.345 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:20.345 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:20.346 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:20.347 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:20.364 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:20.365 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:20.365 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:20.365 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:20.366 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:20.366 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:20.366 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:20.366 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:20.366 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:20.367 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=9, TimeoutTracker(timeout=250, elapsed=222), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [3ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [23ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [39ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [123ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [139ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [141ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [141ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [142ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [201ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [202ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [203ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [203ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [222ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [222ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [222ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [222ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, deferred=Deferred@26395414(state=PENDING, result=null, callback=(continuation of Deferred@210182779 after retry RPC after error@1128529192) -> (continuation of Deferred@130260143 after retry RPC after error@274625842) -> (continuation of Deferred@637471160 after retry RPC after error@2039024274) -> (continuation of Deferred@1647626216 after retry RPC after error@27314553) -> (continuation of Deferred@1548722748 after retry RPC after error@1839464125) -> (continuation of Deferred@703360870 after retry RPC after error@780177443) -> (continuation of Deferred@260016525 after retry RPC after error@2111559881) -> (continuation of Deferred@931536715 after retry RPC after error@497909776), errback=(continuation of Deferred@210182779 after retry RPC after error@1128529192) -> (continuation of Deferred@130260143 after retry RPC after error@274625842) -> (continuation of Deferred@637471160 after retry RPC after error@2039024274) -> (continuation of Deferred@1647626216 after retry RPC after error@27314553) -> (continuation of Deferred@1548722748 after retry RPC after error@1839464125) -> (continuation of Deferred@703360870 after retry RPC after error@780177443) -> (continuation of Deferred@260016525 after retry RPC after error@2111559881) -> (continuation of Deferred@931536715 after retry RPC after error@497909776)))
21:04:20.367 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=9, TimeoutTracker(timeout=250, elapsed=222), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [3ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [23ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [39ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [123ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [139ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [141ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [141ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [142ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [201ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [202ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [203ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [203ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [222ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [222ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [222ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [222ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, deferred=Deferred@26395414(state=PENDING, result=null, callback=(continuation of Deferred@210182779 after retry RPC after error@1128529192) -> (continuation of Deferred@130260143 after retry RPC after error@274625842) -> (continuation of Deferred@637471160 after retry RPC after error@2039024274) -> (continuation of Deferred@1647626216 after retry RPC after error@27314553) -> (continuation of Deferred@1548722748 after retry RPC after error@1839464125) -> (continuation of Deferred@703360870 after retry RPC after error@780177443) -> (continuation of Deferred@260016525 after retry RPC after error@2111559881) -> (continuation of Deferred@931536715 after retry RPC after error@497909776), errback=(continuation of Deferred@210182779 after retry RPC after error@1128529192) -> (continuation of Deferred@130260143 after retry RPC after error@274625842) -> (continuation of Deferred@637471160 after retry RPC after error@2039024274) -> (continuation of Deferred@1647626216 after retry RPC after error@27314553) -> (continuation of Deferred@1548722748 after retry RPC after error@1839464125) -> (continuation of Deferred@703360870 after retry RPC after error@780177443) -> (continuation of Deferred@260016525 after retry RPC after error@2111559881) -> (continuation of Deferred@931536715 after retry RPC after error@497909776)))
21:04:20.384 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:20.385 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:20.385 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:20.385 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:20.385 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:20.386 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:20.386 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:20.386 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:20.387 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:20.404 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:20.405 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:20.405 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:20.406 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:20.406 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:20.406 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:20.406 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:20.406 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:20.407 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:20.424 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:20.425 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:20.425 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:20.425 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:20.426 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:20.426 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:20.426 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:20.426 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:20.426 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:20.444 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:20.445 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:20.445 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:20.446 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:20.446 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:20.446 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:20.446 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:20.447 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:20.447 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:20.464 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:20.465 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:20.465 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:20.465 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:20.466 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:20.466 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:20.466 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:20.466 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:20.467 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:20.484 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:20.485 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:20.485 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:20.486 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:20.486 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:20.486 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:20.486 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:20.486 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:20.487 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:20.544 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:20.544 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:20.545 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:20.545 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:20.545 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:20.545 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:20.545 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:20.545 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:20.546 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:20.625 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:20.625 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:20.625 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:20.626 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:20.626 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:20.626 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:20.626 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:20.626 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:20.627 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:20.627 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=242), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [23ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [63ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [159ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [161ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [161ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [162ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [242ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [242ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [242ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [243ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, deferred=Deferred@804629281(state=PENDING, result=null, callback=(continuation of Deferred@581749180 after retry RPC after error@412463633) -> (continuation of Deferred@440740801 after retry RPC after error@409675531) -> (continuation of Deferred@1548680339 after retry RPC after error@1769916465) -> (continuation of Deferred@1515650556 after retry RPC after error@465885890) -> (continuation of Deferred@809830221 after retry RPC after error@803565108) -> (continuation of Deferred@1118949161 after retry RPC after error@2132676471) -> (continuation of Deferred@1744375990 after retry RPC after error@1667847859), errback=(continuation of Deferred@581749180 after retry RPC after error@412463633) -> (continuation of Deferred@440740801 after retry RPC after error@409675531) -> (continuation of Deferred@1548680339 after retry RPC after error@1769916465) -> (continuation of Deferred@1515650556 after retry RPC after error@465885890) -> (continuation of Deferred@809830221 after retry RPC after error@803565108) -> (continuation of Deferred@1118949161 after retry RPC after error@2132676471) -> (continuation of Deferred@1744375990 after retry RPC after error@1667847859)))
21:04:20.628 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=242), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [23ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [63ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [159ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [161ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [161ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [162ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [242ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [242ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [242ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [243ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, deferred=Deferred@804629281(state=PENDING, result=null, callback=(continuation of Deferred@581749180 after retry RPC after error@412463633) -> (continuation of Deferred@440740801 after retry RPC after error@409675531) -> (continuation of Deferred@1548680339 after retry RPC after error@1769916465) -> (continuation of Deferred@1515650556 after retry RPC after error@465885890) -> (continuation of Deferred@809830221 after retry RPC after error@803565108) -> (continuation of Deferred@1118949161 after retry RPC after error@2132676471) -> (continuation of Deferred@1744375990 after retry RPC after error@1667847859), errback=(continuation of Deferred@581749180 after retry RPC after error@412463633) -> (continuation of Deferred@440740801 after retry RPC after error@409675531) -> (continuation of Deferred@1548680339 after retry RPC after error@1769916465) -> (continuation of Deferred@1515650556 after retry RPC after error@465885890) -> (continuation of Deferred@809830221 after retry RPC after error@803565108) -> (continuation of Deferred@1118949161 after retry RPC after error@2132676471) -> (continuation of Deferred@1744375990 after retry RPC after error@1667847859)))
21:04:20.645 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:20.645 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:20.646 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:20.646 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:20.646 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:20.646 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:20.646 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:20.646 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:20.647 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:20.664 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:20.664 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:20.665 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:20.665 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:20.665 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:20.665 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:20.665 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:20.665 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:20.666 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:20.684 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:20.685 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:20.685 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:20.685 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:20.685 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:20.685 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:20.685 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:20.685 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:20.686 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:20.704 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:20.705 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:20.705 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:20.705 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:20.706 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:20.706 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:20.706 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:20.706 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:20.707 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:20.725 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:20.725 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:20.726 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:20.726 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:20.726 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:20.726 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:20.726 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:20.726 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:20.727 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:20.744 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:20.744 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:20.745 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:20.745 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:20.745 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:20.745 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:20.746 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:20.746 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:20.746 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:20.804 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:20.805 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:20.805 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:20.805 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:20.805 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:20.806 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:20.806 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:20.806 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:20.806 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:20.865 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:20.865 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:20.866 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:20.866 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:20.866 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:20.866 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:20.866 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:20.866 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:20.867 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:20.868 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=222), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [3ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [83ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [99ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [161ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [162ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [162ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [222ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [222ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [223ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [223ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, deferred=Deferred@1334060124(state=PENDING, result=null, callback=(continuation of Deferred@2133266563 after retry RPC after error@432550315) -> (continuation of Deferred@1834815357 after retry RPC after error@60121166) -> (continuation of Deferred@1156697015 after retry RPC after error@485294059) -> (continuation of Deferred@424619500 after retry RPC after error@829467035) -> (continuation of Deferred@2073452657 after retry RPC after error@8309583) -> (continuation of Deferred@1614718689 after retry RPC after error@1412884236) -> (continuation of Deferred@485199891 after retry RPC after error@1782121586), errback=(continuation of Deferred@2133266563 after retry RPC after error@432550315) -> (continuation of Deferred@1834815357 after retry RPC after error@60121166) -> (continuation of Deferred@1156697015 after retry RPC after error@485294059) -> (continuation of Deferred@424619500 after retry RPC after error@829467035) -> (continuation of Deferred@2073452657 after retry RPC after error@8309583) -> (continuation of Deferred@1614718689 after retry RPC after error@1412884236) -> (continuation of Deferred@485199891 after retry RPC after error@1782121586)))
21:04:20.868 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=222), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [3ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [83ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [99ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [161ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [162ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [162ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [222ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [222ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [223ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [223ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, deferred=Deferred@1334060124(state=PENDING, result=null, callback=(continuation of Deferred@2133266563 after retry RPC after error@432550315) -> (continuation of Deferred@1834815357 after retry RPC after error@60121166) -> (continuation of Deferred@1156697015 after retry RPC after error@485294059) -> (continuation of Deferred@424619500 after retry RPC after error@829467035) -> (continuation of Deferred@2073452657 after retry RPC after error@8309583) -> (continuation of Deferred@1614718689 after retry RPC after error@1412884236) -> (continuation of Deferred@485199891 after retry RPC after error@1782121586), errback=(continuation of Deferred@2133266563 after retry RPC after error@432550315) -> (continuation of Deferred@1834815357 after retry RPC after error@60121166) -> (continuation of Deferred@1156697015 after retry RPC after error@485294059) -> (continuation of Deferred@424619500 after retry RPC after error@829467035) -> (continuation of Deferred@2073452657 after retry RPC after error@8309583) -> (continuation of Deferred@1614718689 after retry RPC after error@1412884236) -> (continuation of Deferred@485199891 after retry RPC after error@1782121586)))
21:04:20.884 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:20.884 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:20.885 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:20.885 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:20.885 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:20.885 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:20.885 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:20.885 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:20.886 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:20.904 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:20.905 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:20.905 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:20.905 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:20.905 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:20.906 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:20.906 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:20.906 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:20.906 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:20.924 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:20.925 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:20.925 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:20.925 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:20.925 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:20.925 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:20.925 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:20.925 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:20.926 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:20.944 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:20.945 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:20.945 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:20.945 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:20.945 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:20.945 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:20.945 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:20.946 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:20.947 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:20.964 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:20.965 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:20.965 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:20.965 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:20.965 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:20.966 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:20.966 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:20.966 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:20.967 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:20.984 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:20.985 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:20.985 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:20.985 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:20.985 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:20.985 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:20.986 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:20.986 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:20.986 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:21.004 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:21.005 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:21.005 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:21.005 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:21.005 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:21.006 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:21.006 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:21.006 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:21.006 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:21.085 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:21.085 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:21.086 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:21.086 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:21.086 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:21.086 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:21.086 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:21.086 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:21.086 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:21.104 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:21.105 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:21.105 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:21.105 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:21.106 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:21.106 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:21.106 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:21.106 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:21.106 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:21.107 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=9, TimeoutTracker(timeout=250, elapsed=222), Traces: [0ms] refreshing cache from master, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [3ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [21ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [23ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [23ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [41ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [43ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [61ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [64ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [64ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [81ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [83ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [84ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [84ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [101ms] refreshing cache from master, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [102ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [103ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [103ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [121ms] refreshing cache from master, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [122ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [122ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [123ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [123ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [201ms] refreshing cache from master, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [202ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [202ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [203ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [203ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [203ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [203ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [221ms] refreshing cache from master, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [222ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [222ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [223ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [223ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [223ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [223ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, deferred=Deferred@684865865(state=PENDING, result=null, callback=(continuation of Deferred@1259446276 after retry RPC after error@2016574561) -> (continuation of Deferred@1061377718 after retry RPC after error@115488870) -> (continuation of Deferred@1077262654 after retry RPC after error@730797101) -> (continuation of Deferred@240910604 after retry RPC after error@434776489) -> (continuation of Deferred@1962643427 after retry RPC after error@663147309) -> (continuation of Deferred@1228726960 after retry RPC after error@22232415) -> (continuation of Deferred@696200705 after retry RPC after error@946970552) -> (continuation of Deferred@376882020 after retry RPC after error@240626992), errback=(continuation of Deferred@1259446276 after retry RPC after error@2016574561) -> (continuation of Deferred@1061377718 after retry RPC after error@115488870) -> (continuation of Deferred@1077262654 after retry RPC after error@730797101) -> (continuation of Deferred@240910604 after retry RPC after error@434776489) -> (continuation of Deferred@1962643427 after retry RPC after error@663147309) -> (continuation of Deferred@1228726960 after retry RPC after error@22232415) -> (continuation of Deferred@696200705 after retry RPC after error@946970552) -> (continuation of Deferred@376882020 after retry RPC after error@240626992)))
21:04:21.108 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=9, TimeoutTracker(timeout=250, elapsed=222), Traces: [0ms] refreshing cache from master, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [3ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [21ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [23ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [23ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [41ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [43ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [61ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [64ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [64ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [81ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [83ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [84ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [84ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [101ms] refreshing cache from master, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [102ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [103ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [103ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [121ms] refreshing cache from master, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [122ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [122ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [123ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [123ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [201ms] refreshing cache from master, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [202ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [202ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [203ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [203ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [203ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [203ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [221ms] refreshing cache from master, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [222ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [222ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [223ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [223ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [223ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [223ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, deferred=Deferred@684865865(state=PENDING, result=null, callback=(continuation of Deferred@1259446276 after retry RPC after error@2016574561) -> (continuation of Deferred@1061377718 after retry RPC after error@115488870) -> (continuation of Deferred@1077262654 after retry RPC after error@730797101) -> (continuation of Deferred@240910604 after retry RPC after error@434776489) -> (continuation of Deferred@1962643427 after retry RPC after error@663147309) -> (continuation of Deferred@1228726960 after retry RPC after error@22232415) -> (continuation of Deferred@696200705 after retry RPC after error@946970552) -> (continuation of Deferred@376882020 after retry RPC after error@240626992), errback=(continuation of Deferred@1259446276 after retry RPC after error@2016574561) -> (continuation of Deferred@1061377718 after retry RPC after error@115488870) -> (continuation of Deferred@1077262654 after retry RPC after error@730797101) -> (continuation of Deferred@240910604 after retry RPC after error@434776489) -> (continuation of Deferred@1962643427 after retry RPC after error@663147309) -> (continuation of Deferred@1228726960 after retry RPC after error@22232415) -> (continuation of Deferred@696200705 after retry RPC after error@946970552) -> (continuation of Deferred@376882020 after retry RPC after error@240626992)))
21:04:21.124 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:21.125 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:21.125 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:21.125 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:21.125 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:21.125 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:21.126 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:21.126 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:21.126 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:21.144 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:21.145 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:21.145 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:21.145 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:21.145 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:21.145 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:21.145 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:21.145 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:21.146 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:21.164 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:21.165 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:21.165 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:21.165 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:21.165 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:21.165 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:21.165 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:21.165 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:21.166 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:21.184 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:21.185 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:21.185 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:21.185 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:21.185 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:21.185 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:21.185 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:21.185 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:21.186 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:21.204 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:21.205 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:21.205 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:21.205 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:21.205 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:21.206 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:21.206 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:21.206 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:21.206 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:21.244 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:21.244 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:21.245 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:21.245 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:21.245 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:21.245 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:21.245 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:21.245 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:21.246 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:21.284 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:21.284 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:21.285 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:21.285 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:21.285 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:21.285 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:21.285 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:21.285 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:21.286 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:21.287 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=162), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [119ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [161ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [161ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [162ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, deferred=Deferred@177036163(state=PENDING, result=null, callback=(continuation of Deferred@1859334950 after retry RPC after error@1991545066) -> (continuation of Deferred@800505005 after retry RPC after error@1325906843) -> (continuation of Deferred@951778142 after retry RPC after error@1784356707) -> (continuation of Deferred@1880266065 after retry RPC after error@45818795) -> (continuation of Deferred@1840531625 after retry RPC after error@1604772475) -> (continuation of Deferred@1006263491 after retry RPC after error@1619494508), errback=(continuation of Deferred@1859334950 after retry RPC after error@1991545066) -> (continuation of Deferred@800505005 after retry RPC after error@1325906843) -> (continuation of Deferred@951778142 after retry RPC after error@1784356707) -> (continuation of Deferred@1880266065 after retry RPC after error@45818795) -> (continuation of Deferred@1840531625 after retry RPC after error@1604772475) -> (continuation of Deferred@1006263491 after retry RPC after error@1619494508)))
21:04:21.287 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=162), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [119ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [161ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [161ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [162ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, deferred=Deferred@177036163(state=PENDING, result=null, callback=(continuation of Deferred@1859334950 after retry RPC after error@1991545066) -> (continuation of Deferred@800505005 after retry RPC after error@1325906843) -> (continuation of Deferred@951778142 after retry RPC after error@1784356707) -> (continuation of Deferred@1880266065 after retry RPC after error@45818795) -> (continuation of Deferred@1840531625 after retry RPC after error@1604772475) -> (continuation of Deferred@1006263491 after retry RPC after error@1619494508), errback=(continuation of Deferred@1859334950 after retry RPC after error@1991545066) -> (continuation of Deferred@800505005 after retry RPC after error@1325906843) -> (continuation of Deferred@951778142 after retry RPC after error@1784356707) -> (continuation of Deferred@1880266065 after retry RPC after error@45818795) -> (continuation of Deferred@1840531625 after retry RPC after error@1604772475) -> (continuation of Deferred@1006263491 after retry RPC after error@1619494508)))
21:04:21.304 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:21.305 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:21.305 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:21.305 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:21.305 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:21.305 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:21.305 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:21.305 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:21.307 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:21.324 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:21.325 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:21.325 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:21.325 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:21.325 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:21.325 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:21.325 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:21.325 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:21.326 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:21.344 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:21.344 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:21.345 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:21.345 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:21.345 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:21.345 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:21.345 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:21.345 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:21.346 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:21.364 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:21.365 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:21.365 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:21.365 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:21.365 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:21.365 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:21.366 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:21.366 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:21.366 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:21.384 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:21.385 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:21.385 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:21.385 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:21.385 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:21.385 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:21.385 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:21.385 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:21.386 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:21.404 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:21.405 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:21.405 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:21.405 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:21.405 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:21.405 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:21.405 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:21.405 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:21.406 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:21.424 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:21.425 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:21.425 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:21.425 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:21.425 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:21.425 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:21.425 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:21.425 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:21.426 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:21.505 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:21.505 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:21.505 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:21.506 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:21.506 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:21.506 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:21.506 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:21.506 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:21.507 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:21.507 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=203), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [3ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [202ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [202ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [203ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [203ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, deferred=Deferred@1414186954(state=PENDING, result=null, callback=(continuation of Deferred@1005898981 after retry RPC after error@1585079102) -> (continuation of Deferred@270410190 after retry RPC after error@317243536) -> (continuation of Deferred@266773964 after retry RPC after error@42488989) -> (continuation of Deferred@1668879699 after retry RPC after error@927049196) -> (continuation of Deferred@229879309 after retry RPC after error@1306476442) -> (continuation of Deferred@1713858815 after retry RPC after error@1551093522) -> (continuation of Deferred@1277215063 after retry RPC after error@2025264294), errback=(continuation of Deferred@1005898981 after retry RPC after error@1585079102) -> (continuation of Deferred@270410190 after retry RPC after error@317243536) -> (continuation of Deferred@266773964 after retry RPC after error@42488989) -> (continuation of Deferred@1668879699 after retry RPC after error@927049196) -> (continuation of Deferred@229879309 after retry RPC after error@1306476442) -> (continuation of Deferred@1713858815 after retry RPC after error@1551093522) -> (continuation of Deferred@1277215063 after retry RPC after error@2025264294)))
21:04:21.508 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=203), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [3ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [202ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [202ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [203ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [203ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, deferred=Deferred@1414186954(state=PENDING, result=null, callback=(continuation of Deferred@1005898981 after retry RPC after error@1585079102) -> (continuation of Deferred@270410190 after retry RPC after error@317243536) -> (continuation of Deferred@266773964 after retry RPC after error@42488989) -> (continuation of Deferred@1668879699 after retry RPC after error@927049196) -> (continuation of Deferred@229879309 after retry RPC after error@1306476442) -> (continuation of Deferred@1713858815 after retry RPC after error@1551093522) -> (continuation of Deferred@1277215063 after retry RPC after error@2025264294), errback=(continuation of Deferred@1005898981 after retry RPC after error@1585079102) -> (continuation of Deferred@270410190 after retry RPC after error@317243536) -> (continuation of Deferred@266773964 after retry RPC after error@42488989) -> (continuation of Deferred@1668879699 after retry RPC after error@927049196) -> (continuation of Deferred@229879309 after retry RPC after error@1306476442) -> (continuation of Deferred@1713858815 after retry RPC after error@1551093522) -> (continuation of Deferred@1277215063 after retry RPC after error@2025264294)))
21:04:21.525 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:21.525 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:21.526 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:21.526 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:21.526 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:21.526 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:21.526 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:21.526 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:21.527 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:21.545 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:21.545 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:21.546 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:21.546 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:21.546 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:21.546 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:21.546 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:21.546 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:21.547 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:21.565 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:21.565 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:21.566 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:21.566 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:21.566 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:21.566 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:21.566 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:21.566 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:21.567 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:21.584 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:21.584 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:21.585 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:21.585 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:21.585 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:21.585 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:21.585 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:21.585 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:21.586 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:21.604 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:21.605 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:21.605 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:21.605 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:21.605 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:21.606 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:21.606 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:21.606 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:21.607 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:21.644 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:21.645 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:21.645 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:21.645 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:21.645 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:21.645 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:21.646 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:21.646 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:21.647 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:21.664 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:21.665 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:21.665 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:21.665 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:21.665 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:21.665 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:21.665 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:21.666 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:21.666 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:21.704 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:21.705 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:21.705 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:21.705 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:21.705 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:21.706 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:21.706 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:21.706 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:21.706 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:21.707 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=182), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [3ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [23ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [43ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [59ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [79ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [83ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [141ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [141ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [142ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [182ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [182ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [182ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [183ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, deferred=Deferred@1966213727(state=PENDING, result=null, callback=(continuation of Deferred@1547631554 after retry RPC after error@585036297) -> (continuation of Deferred@439525567 after retry RPC after error@2134323540) -> (continuation of Deferred@1417426331 after retry RPC after error@23583655) -> (continuation of Deferred@249537644 after retry RPC after error@1218742642) -> (continuation of Deferred@2101456144 after retry RPC after error@1111813116) -> (continuation of Deferred@827054876 after retry RPC after error@1565014878) -> (continuation of Deferred@764490954 after retry RPC after error@1223635343), errback=(continuation of Deferred@1547631554 after retry RPC after error@585036297) -> (continuation of Deferred@439525567 after retry RPC after error@2134323540) -> (continuation of Deferred@1417426331 after retry RPC after error@23583655) -> (continuation of Deferred@249537644 after retry RPC after error@1218742642) -> (continuation of Deferred@2101456144 after retry RPC after error@1111813116) -> (continuation of Deferred@827054876 after retry RPC after error@1565014878) -> (continuation of Deferred@764490954 after retry RPC after error@1223635343)))
21:04:21.708 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=182), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [3ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [23ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [43ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [59ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [79ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [83ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [141ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [141ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [142ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [182ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [182ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [182ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [183ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, deferred=Deferred@1966213727(state=PENDING, result=null, callback=(continuation of Deferred@1547631554 after retry RPC after error@585036297) -> (continuation of Deferred@439525567 after retry RPC after error@2134323540) -> (continuation of Deferred@1417426331 after retry RPC after error@23583655) -> (continuation of Deferred@249537644 after retry RPC after error@1218742642) -> (continuation of Deferred@2101456144 after retry RPC after error@1111813116) -> (continuation of Deferred@827054876 after retry RPC after error@1565014878) -> (continuation of Deferred@764490954 after retry RPC after error@1223635343), errback=(continuation of Deferred@1547631554 after retry RPC after error@585036297) -> (continuation of Deferred@439525567 after retry RPC after error@2134323540) -> (continuation of Deferred@1417426331 after retry RPC after error@23583655) -> (continuation of Deferred@249537644 after retry RPC after error@1218742642) -> (continuation of Deferred@2101456144 after retry RPC after error@1111813116) -> (continuation of Deferred@827054876 after retry RPC after error@1565014878) -> (continuation of Deferred@764490954 after retry RPC after error@1223635343)))
21:04:21.724 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:21.725 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:21.725 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:21.725 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:21.726 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:21.726 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:21.726 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:21.726 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:21.728 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:21.744 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:21.745 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:21.745 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:21.745 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:21.745 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:21.745 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:21.746 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:21.746 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:21.746 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:21.764 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:21.765 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:21.766 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:21.766 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:21.766 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:21.766 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:21.766 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:21.766 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:21.767 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:21.784 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:21.785 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:21.786 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:21.786 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:21.786 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:21.786 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:21.786 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:21.786 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:21.786 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:21.805 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:21.805 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:21.806 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:21.806 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:21.806 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:21.806 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:21.806 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:21.806 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:21.806 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:21.844 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:21.844 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:21.845 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:21.845 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:21.845 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:21.846 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:21.846 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:21.846 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:21.846 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:21.904 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:21.905 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:21.905 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:21.905 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:21.905 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:21.906 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:21.906 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:21.906 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:21.907 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:21.965 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:21.965 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:21.966 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:21.966 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:21.966 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:21.966 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:21.966 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:21.966 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:21.967 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:21.967 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=242), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [4ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [4ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [43ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [119ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [122ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [182ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [182ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [182ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [183ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [242ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [242ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [243ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [243ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, deferred=Deferred@864005044(state=PENDING, result=null, callback=(continuation of Deferred@619517876 after retry RPC after error@244992401) -> (continuation of Deferred@1808901267 after retry RPC after error@633176150) -> (continuation of Deferred@1763795938 after retry RPC after error@900426134) -> (continuation of Deferred@1404614284 after retry RPC after error@449721245) -> (continuation of Deferred@2014473557 after retry RPC after error@370891800) -> (continuation of Deferred@8492161 after retry RPC after error@1798839767) -> (continuation of Deferred@11255072 after retry RPC after error@1276959409), errback=(continuation of Deferred@619517876 after retry RPC after error@244992401) -> (continuation of Deferred@1808901267 after retry RPC after error@633176150) -> (continuation of Deferred@1763795938 after retry RPC after error@900426134) -> (continuation of Deferred@1404614284 after retry RPC after error@449721245) -> (continuation of Deferred@2014473557 after retry RPC after error@370891800) -> (continuation of Deferred@8492161 after retry RPC after error@1798839767) -> (continuation of Deferred@11255072 after retry RPC after error@1276959409)))
21:04:21.968 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=242), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [4ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [4ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [43ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [119ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [122ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [182ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [182ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [182ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [183ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [242ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [242ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [243ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [243ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, deferred=Deferred@864005044(state=PENDING, result=null, callback=(continuation of Deferred@619517876 after retry RPC after error@244992401) -> (continuation of Deferred@1808901267 after retry RPC after error@633176150) -> (continuation of Deferred@1763795938 after retry RPC after error@900426134) -> (continuation of Deferred@1404614284 after retry RPC after error@449721245) -> (continuation of Deferred@2014473557 after retry RPC after error@370891800) -> (continuation of Deferred@8492161 after retry RPC after error@1798839767) -> (continuation of Deferred@11255072 after retry RPC after error@1276959409), errback=(continuation of Deferred@619517876 after retry RPC after error@244992401) -> (continuation of Deferred@1808901267 after retry RPC after error@633176150) -> (continuation of Deferred@1763795938 after retry RPC after error@900426134) -> (continuation of Deferred@1404614284 after retry RPC after error@449721245) -> (continuation of Deferred@2014473557 after retry RPC after error@370891800) -> (continuation of Deferred@8492161 after retry RPC after error@1798839767) -> (continuation of Deferred@11255072 after retry RPC after error@1276959409)))
21:04:21.984 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:21.984 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:21.985 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:21.985 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:21.985 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:21.985 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:21.985 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:21.985 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:21.986 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:22.004 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:22.004 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:22.005 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:22.005 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:22.005 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:22.005 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:22.005 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:22.005 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:22.006 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:22.024 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:22.024 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:22.025 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:22.025 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:22.025 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:22.025 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:22.025 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:22.025 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:22.026 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:22.044 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:22.044 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:22.045 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:22.045 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:22.045 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:22.045 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:22.045 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:22.045 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:22.046 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:22.064 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:22.065 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:22.065 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:22.065 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:22.065 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:22.065 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:22.066 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:22.066 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:22.066 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:22.104 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:22.105 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:22.105 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:22.105 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:22.105 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:22.105 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:22.106 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:22.106 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:22.106 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:22.144 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:22.145 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:22.145 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:22.146 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:22.146 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:22.146 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:22.146 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:22.146 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:22.147 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:22.184 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:22.184 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:22.185 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:22.185 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:22.185 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:22.185 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:22.185 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:22.185 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:22.186 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:22.187 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=203), Traces: [0ms] refreshing cache from master, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [3ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [21ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [23ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [41ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [43ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [61ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [63ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [81ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [83ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [83ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [121ms] refreshing cache from master, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [122ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [122ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [123ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [123ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [124ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [161ms] refreshing cache from master, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [162ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [162ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [163ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [163ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [163ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [164ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [200ms] refreshing cache from master, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [202ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [202ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [203ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [204ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, deferred=Deferred@945424095(state=PENDING, result=null, callback=(continuation of Deferred@1774393548 after retry RPC after error@622299968) -> (continuation of Deferred@72332978 after retry RPC after error@1816863770) -> (continuation of Deferred@490689575 after retry RPC after error@82547019) -> (continuation of Deferred@2103991618 after retry RPC after error@1211555073) -> (continuation of Deferred@793720396 after retry RPC after error@2012650279) -> (continuation of Deferred@1221282518 after retry RPC after error@913000475) -> (continuation of Deferred@1623254476 after retry RPC after error@598500688), errback=(continuation of Deferred@1774393548 after retry RPC after error@622299968) -> (continuation of Deferred@72332978 after retry RPC after error@1816863770) -> (continuation of Deferred@490689575 after retry RPC after error@82547019) -> (continuation of Deferred@2103991618 after retry RPC after error@1211555073) -> (continuation of Deferred@793720396 after retry RPC after error@2012650279) -> (continuation of Deferred@1221282518 after retry RPC after error@913000475) -> (continuation of Deferred@1623254476 after retry RPC after error@598500688)))
21:04:22.188 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=203), Traces: [0ms] refreshing cache from master, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [3ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [21ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [23ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [41ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [43ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [61ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [63ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [81ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [83ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [83ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [121ms] refreshing cache from master, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [122ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [122ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [123ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [123ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [124ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [161ms] refreshing cache from master, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [162ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [162ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [163ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [163ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [163ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [164ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [200ms] refreshing cache from master, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [202ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [202ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [203ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [204ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, deferred=Deferred@945424095(state=PENDING, result=null, callback=(continuation of Deferred@1774393548 after retry RPC after error@622299968) -> (continuation of Deferred@72332978 after retry RPC after error@1816863770) -> (continuation of Deferred@490689575 after retry RPC after error@82547019) -> (continuation of Deferred@2103991618 after retry RPC after error@1211555073) -> (continuation of Deferred@793720396 after retry RPC after error@2012650279) -> (continuation of Deferred@1221282518 after retry RPC after error@913000475) -> (continuation of Deferred@1623254476 after retry RPC after error@598500688), errback=(continuation of Deferred@1774393548 after retry RPC after error@622299968) -> (continuation of Deferred@72332978 after retry RPC after error@1816863770) -> (continuation of Deferred@490689575 after retry RPC after error@82547019) -> (continuation of Deferred@2103991618 after retry RPC after error@1211555073) -> (continuation of Deferred@793720396 after retry RPC after error@2012650279) -> (continuation of Deferred@1221282518 after retry RPC after error@913000475) -> (continuation of Deferred@1623254476 after retry RPC after error@598500688)))
21:04:22.204 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:22.204 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:22.205 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:22.205 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:22.205 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:22.206 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:22.206 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:22.206 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:22.207 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:22.224 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:22.225 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:22.225 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:22.225 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:22.225 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:22.225 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:22.225 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:22.225 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:22.226 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:22.244 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:22.245 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:22.245 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:22.245 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:22.245 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:22.245 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:22.245 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:22.245 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:22.246 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:22.264 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:22.265 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:22.265 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:22.265 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:22.265 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:22.265 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:22.265 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:22.265 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:22.266 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:22.284 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:22.285 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:22.285 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:22.285 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:22.285 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:22.286 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:22.286 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:22.286 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:22.287 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:22.304 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:22.305 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:22.305 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:22.305 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:22.305 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:22.306 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:22.306 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:22.306 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:22.307 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:22.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:22.324891 24262 raft_consensus.cc:493] T 00000000000000000000000000000000 P eb664add87f44013b596f4cc01c9a41c [term 4 FOLLOWER]: Starting pre-election (detected failure of leader c520077acf3341e29447c6c2b8d3f74c)
21:04:22.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:22.325109 24262 raft_consensus.cc:515] T 00000000000000000000000000000000 P eb664add87f44013b596f4cc01c9a41c [term 4 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e9dd218ca2244f5a934aac2b7e58eb1d" member_type: VOTER last_known_addr { host: "127.22.168.62" port: 37239 } } peers { permanent_uuid: "c520077acf3341e29447c6c2b8d3f74c" member_type: VOTER last_known_addr { host: "127.22.168.61" port: 32947 } } peers { permanent_uuid: "eb664add87f44013b596f4cc01c9a41c" member_type: VOTER last_known_addr { host: "127.22.168.60" port: 33247 } }
21:04:22.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:22.325911 24262 leader_election.cc:290] T 00000000000000000000000000000000 P eb664add87f44013b596f4cc01c9a41c [CANDIDATE]: Term 5 pre-election: Requested pre-vote from peers e9dd218ca2244f5a934aac2b7e58eb1d (127.22.168.62:37239), c520077acf3341e29447c6c2b8d3f74c (127.22.168.61:32947)
21:04:22.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:22.327999 24086 leader_election.cc:336] T 00000000000000000000000000000000 P eb664add87f44013b596f4cc01c9a41c [CANDIDATE]: Term 5 pre-election: RPC error from VoteRequest() call to peer e9dd218ca2244f5a934aac2b7e58eb1d (127.22.168.62:37239): Network error: Client connection negotiation failed: client connection to 127.22.168.62:37239: connect: Connection refused (error 111)
21:04:22.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:22.328207 24089 leader_election.cc:336] T 00000000000000000000000000000000 P eb664add87f44013b596f4cc01c9a41c [CANDIDATE]: Term 5 pre-election: RPC error from VoteRequest() call to peer c520077acf3341e29447c6c2b8d3f74c (127.22.168.61:32947): Network error: Client connection negotiation failed: client connection to 127.22.168.61:32947: connect: Connection refused (error 111)
21:04:22.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:22.328387 24089 leader_election.cc:304] T 00000000000000000000000000000000 P eb664add87f44013b596f4cc01c9a41c [CANDIDATE]: Term 5 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: eb664add87f44013b596f4cc01c9a41c; no voters: c520077acf3341e29447c6c2b8d3f74c, e9dd218ca2244f5a934aac2b7e58eb1d
21:04:22.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:22.328704 24262 raft_consensus.cc:2749] T 00000000000000000000000000000000 P eb664add87f44013b596f4cc01c9a41c [term 4 FOLLOWER]: Leader pre-election lost for term 5. Reason: could not achieve majority
21:04:22.364 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:22.365 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:22.365 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:22.365 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:22.366 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:22.366 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:22.366 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:22.366 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:22.367 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:22.405 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:22.405 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:22.406 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:22.406 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:22.406 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:22.406 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:22.406 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:22.406 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:22.407 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:22.444 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:22.444 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:22.445 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:22.445 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:22.445 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:22.445 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:22.445 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:22.445 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:22.447 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:22.447 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=9, TimeoutTracker(timeout=250, elapsed=243), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [3ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [162ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [162ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [162ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [163ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [202ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [202ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [202ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [203ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [241ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [241ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [242ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [243ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, deferred=Deferred@561289423(state=PENDING, result=null, callback=(continuation of Deferred@1271015071 after retry RPC after error@604763117) -> (continuation of Deferred@1394755677 after retry RPC after error@1452609558) -> (continuation of Deferred@213463685 after retry RPC after error@2002136164) -> (continuation of Deferred@913830515 after retry RPC after error@1218203915) -> (continuation of Deferred@234123143 after retry RPC after error@1267160913) -> (continuation of Deferred@1114296376 after retry RPC after error@1758774647) -> (continuation of Deferred@1190266813 after retry RPC after error@935259629) -> (continuation of Deferred@591466546 after retry RPC after error@1844483664), errback=(continuation of Deferred@1271015071 after retry RPC after error@604763117) -> (continuation of Deferred@1394755677 after retry RPC after error@1452609558) -> (continuation of Deferred@213463685 after retry RPC after error@2002136164) -> (continuation of Deferred@913830515 after retry RPC after error@1218203915) -> (continuation of Deferred@234123143 after retry RPC after error@1267160913) -> (continuation of Deferred@1114296376 after retry RPC after error@1758774647) -> (continuation of Deferred@1190266813 after retry RPC after error@935259629) -> (continuation of Deferred@591466546 after retry RPC after error@1844483664)))
21:04:22.448 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=9, TimeoutTracker(timeout=250, elapsed=243), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [3ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [162ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [162ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [162ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [163ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [202ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [202ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [202ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [203ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [241ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [241ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [242ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [243ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, deferred=Deferred@561289423(state=PENDING, result=null, callback=(continuation of Deferred@1271015071 after retry RPC after error@604763117) -> (continuation of Deferred@1394755677 after retry RPC after error@1452609558) -> (continuation of Deferred@213463685 after retry RPC after error@2002136164) -> (continuation of Deferred@913830515 after retry RPC after error@1218203915) -> (continuation of Deferred@234123143 after retry RPC after error@1267160913) -> (continuation of Deferred@1114296376 after retry RPC after error@1758774647) -> (continuation of Deferred@1190266813 after retry RPC after error@935259629) -> (continuation of Deferred@591466546 after retry RPC after error@1844483664), errback=(continuation of Deferred@1271015071 after retry RPC after error@604763117) -> (continuation of Deferred@1394755677 after retry RPC after error@1452609558) -> (continuation of Deferred@213463685 after retry RPC after error@2002136164) -> (continuation of Deferred@913830515 after retry RPC after error@1218203915) -> (continuation of Deferred@234123143 after retry RPC after error@1267160913) -> (continuation of Deferred@1114296376 after retry RPC after error@1758774647) -> (continuation of Deferred@1190266813 after retry RPC after error@935259629) -> (continuation of Deferred@591466546 after retry RPC after error@1844483664)))
21:04:22.464 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:22.464 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:22.465 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:22.465 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:22.465 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:22.465 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:22.465 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:22.465 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:22.466 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:22.484 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:22.484 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:22.485 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:22.485 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:22.485 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:22.485 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:22.485 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:22.485 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:22.486 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:22.504 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:22.504 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:22.505 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:22.505 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:22.505 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:22.505 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:22.505 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:22.505 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:22.506 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:22.524 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:22.524 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:22.525 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:22.525 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:22.525 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:22.525 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:22.525 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:22.525 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:22.526 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:22.544 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:22.544 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:22.545 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:22.545 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:22.545 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:22.545 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:22.545 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:22.545 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:22.546 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:22.584 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:22.585 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:22.585 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:22.585 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:22.585 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:22.585 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:22.585 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:22.585 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:22.586 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:22.604 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:22.605 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:22.605 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:22.605 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:22.605 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:22.605 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:22.606 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:22.606 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:22.607 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:22.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:22.653065 23569 master_proxy_rpc.cc:177] Failed to determine new leader Master: Timed out: ConnectToClusterRpc(addrs: 127.22.168.62:37239,127.22.168.61:32947,127.22.168.60:33247, num_attempts: 97) passed its deadline: Not found: no leader found: ConnectToClusterRpc(addrs: 127.22.168.62:37239,127.22.168.61:32947,127.22.168.60:33247, num_attempts: 1)
21:04:22.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:22.654542 23569 meta_cache.cc:788] Timed out: LookupRpcById { tablet: '01aa0e192c434b468ec7ed90d48563b3', attempt: 2 } failed: LookupRpcById timed out after deadline expired: LookupRpcById { tablet: '01aa0e192c434b468ec7ed90d48563b3', attempt: 2 } passed its deadline: Timed out: ConnectToClusterRpc(addrs: 127.22.168.62:37239,127.22.168.61:32947,127.22.168.60:33247, num_attempts: 97) passed its deadline: Not found: no leader found: ConnectToClusterRpc(addrs: 127.22.168.62:37239,127.22.168.61:32947,127.22.168.60:33247, num_attempts: 1)
21:04:22.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:22.656538 23569 master_proxy_rpc.cc:203] Re-attempting LookupRpcById request to leader Master (127.22.168.60:33247)
21:04:22.704 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:22.704 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:22.705 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:22.705 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:22.705 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:22.705 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:22.705 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:22.705 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:22.706 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:22.706 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=242), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [142ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [142ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [142ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [143ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [241ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [241ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [242ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [242ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, deferred=Deferred@467895969(state=PENDING, result=null, callback=(continuation of Deferred@1704347388 after retry RPC after error@1426750424) -> (continuation of Deferred@1734821995 after retry RPC after error@1614154920) -> (continuation of Deferred@935331828 after retry RPC after error@823291503) -> (continuation of Deferred@1827363894 after retry RPC after error@1013472847) -> (continuation of Deferred@195035537 after retry RPC after error@694384540) -> (continuation of Deferred@534742122 after retry RPC after error@1631271928) -> (continuation of Deferred@1788613319 after retry RPC after error@1492211615), errback=(continuation of Deferred@1704347388 after retry RPC after error@1426750424) -> (continuation of Deferred@1734821995 after retry RPC after error@1614154920) -> (continuation of Deferred@935331828 after retry RPC after error@823291503) -> (continuation of Deferred@1827363894 after retry RPC after error@1013472847) -> (continuation of Deferred@195035537 after retry RPC after error@694384540) -> (continuation of Deferred@534742122 after retry RPC after error@1631271928) -> (continuation of Deferred@1788613319 after retry RPC after error@1492211615)))
21:04:22.707 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=242), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [142ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [142ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [142ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [143ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [241ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [241ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [242ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [242ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, deferred=Deferred@467895969(state=PENDING, result=null, callback=(continuation of Deferred@1704347388 after retry RPC after error@1426750424) -> (continuation of Deferred@1734821995 after retry RPC after error@1614154920) -> (continuation of Deferred@935331828 after retry RPC after error@823291503) -> (continuation of Deferred@1827363894 after retry RPC after error@1013472847) -> (continuation of Deferred@195035537 after retry RPC after error@694384540) -> (continuation of Deferred@534742122 after retry RPC after error@1631271928) -> (continuation of Deferred@1788613319 after retry RPC after error@1492211615), errback=(continuation of Deferred@1704347388 after retry RPC after error@1426750424) -> (continuation of Deferred@1734821995 after retry RPC after error@1614154920) -> (continuation of Deferred@935331828 after retry RPC after error@823291503) -> (continuation of Deferred@1827363894 after retry RPC after error@1013472847) -> (continuation of Deferred@195035537 after retry RPC after error@694384540) -> (continuation of Deferred@534742122 after retry RPC after error@1631271928) -> (continuation of Deferred@1788613319 after retry RPC after error@1492211615)))
21:04:22.724 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:22.724 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:22.725 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:22.725 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:22.725 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:22.725 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:22.725 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:22.725 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:22.726 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:22.744 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:22.744 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:22.745 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:22.745 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:22.745 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:22.745 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:22.745 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:22.745 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:22.746 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:22.764 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:22.764 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:22.765 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:22.765 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:22.765 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:22.765 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:22.765 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:22.765 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:22.766 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:22.784 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:22.784 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:22.785 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:22.785 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:22.785 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:22.785 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:22.785 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:22.785 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:22.786 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:22.804 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:22.804 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:22.805 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:22.805 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:22.805 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:22.805 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:22.805 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:22.805 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:22.806 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:22.824 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:22.824 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:22.825 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:22.825 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:22.825 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:22.825 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:22.825 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:22.825 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:22.826 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:22.864 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:22.865 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:22.865 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:22.865 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:22.865 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:22.865 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:22.866 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:22.866 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:22.866 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:22.984 [DEBUG - pool-40-thread-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=260), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [141ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [141ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [142ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, deferred=Deferred@297307282(state=PENDING, result=null, callback=(continuation of Deferred@586486907 after retry RPC after error@875865749) -> (continuation of Deferred@843491419 after retry RPC after error@1733263005) -> (continuation of Deferred@1838409923 after retry RPC after error@2088958954) -> (continuation of Deferred@2063700049 after retry RPC after error@1738977850) -> (continuation of Deferred@352427005 after retry RPC after error@2125950748) -> (continuation of Deferred@155446147 after retry RPC after error@1529262067) -> (continuation of Deferred@1122048281 after retry RPC after error@124469718), errback=(continuation of Deferred@586486907 after retry RPC after error@875865749) -> (continuation of Deferred@843491419 after retry RPC after error@1733263005) -> (continuation of Deferred@1838409923 after retry RPC after error@2088958954) -> (continuation of Deferred@2063700049 after retry RPC after error@1738977850) -> (continuation of Deferred@352427005 after retry RPC after error@2125950748) -> (continuation of Deferred@155446147 after retry RPC after error@1529262067) -> (continuation of Deferred@1122048281 after retry RPC after error@124469718)))
21:04:22.985 [WARN - pool-40-thread-1] (HashedWheelTimer.java:716) An exception was thrown by TimerTask.
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343) ~[main/:?]
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at java.lang.Thread.run(Thread.java:748) [?:1.8.0_275]
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86)
at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106)
at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198)
at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158)
at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953)
at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923)
at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464)
at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713)
at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701)
at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788)
at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501)
at java.lang.Thread.run(Thread.java:748)
21:04:23.004 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:23.004 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:23.005 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:23.005 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:23.005 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:23.005 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:23.005 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:23.005 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:23.006 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:23.024 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:23.025 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:23.025 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:23.025 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:23.025 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:23.026 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:23.026 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:23.026 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:23.027 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:23.044 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:23.045 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:23.045 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:23.045 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:23.045 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:23.045 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:23.046 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:23.046 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:23.046 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:23.064 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:23.065 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:23.065 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:23.066 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:23.066 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:23.066 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:23.066 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:23.066 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:23.067 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:23.084 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:23.085 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:23.086 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:23.086 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:23.086 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:23.086 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:23.086 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:23.086 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:23.087 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:23.104 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:23.105 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:23.106 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:23.106 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:23.106 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:23.106 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:23.106 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:23.106 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:23.107 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:23.144 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:23.144 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:23.145 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:23.145 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:23.145 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:23.145 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:23.145 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:23.145 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:23.146 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:23.146 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=142), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [103ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [139ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [141ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [141ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [141ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, deferred=Deferred@1415174675(state=PENDING, result=null, callback=(continuation of Deferred@118278940 after retry RPC after error@1316246177) -> (continuation of Deferred@759456674 after retry RPC after error@928492891) -> (continuation of Deferred@1449313414 after retry RPC after error@980379057) -> (continuation of Deferred@876103725 after retry RPC after error@1041172100) -> (continuation of Deferred@2122953527 after retry RPC after error@1066009530) -> (continuation of Deferred@1225084230 after retry RPC after error@259545540), errback=(continuation of Deferred@118278940 after retry RPC after error@1316246177) -> (continuation of Deferred@759456674 after retry RPC after error@928492891) -> (continuation of Deferred@1449313414 after retry RPC after error@980379057) -> (continuation of Deferred@876103725 after retry RPC after error@1041172100) -> (continuation of Deferred@2122953527 after retry RPC after error@1066009530) -> (continuation of Deferred@1225084230 after retry RPC after error@259545540)))
21:04:23.147 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=142), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [103ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [139ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [141ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [141ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [141ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, deferred=Deferred@1415174675(state=PENDING, result=null, callback=(continuation of Deferred@118278940 after retry RPC after error@1316246177) -> (continuation of Deferred@759456674 after retry RPC after error@928492891) -> (continuation of Deferred@1449313414 after retry RPC after error@980379057) -> (continuation of Deferred@876103725 after retry RPC after error@1041172100) -> (continuation of Deferred@2122953527 after retry RPC after error@1066009530) -> (continuation of Deferred@1225084230 after retry RPC after error@259545540), errback=(continuation of Deferred@118278940 after retry RPC after error@1316246177) -> (continuation of Deferred@759456674 after retry RPC after error@928492891) -> (continuation of Deferred@1449313414 after retry RPC after error@980379057) -> (continuation of Deferred@876103725 after retry RPC after error@1041172100) -> (continuation of Deferred@2122953527 after retry RPC after error@1066009530) -> (continuation of Deferred@1225084230 after retry RPC after error@259545540)))
21:04:23.164 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:23.164 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:23.165 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:23.165 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:23.165 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:23.165 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:23.165 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:23.165 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:23.166 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:23.184 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:23.184 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:23.185 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:23.185 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:23.185 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:23.185 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:23.185 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:23.185 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:23.187 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:23.204 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:23.204 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:23.205 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:23.205 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:23.205 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:23.205 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:23.205 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:23.205 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:23.206 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:23.224 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:23.224 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:23.225 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:23.225 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:23.225 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:23.225 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:23.225 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:23.225 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:23.226 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:23.244 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:23.245 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:23.245 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:23.245 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:23.245 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:23.245 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:23.245 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:23.245 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:23.246 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:23.264 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:23.265 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:23.265 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:23.265 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:23.265 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:23.265 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:23.265 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:23.265 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:23.266 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:23.324 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:23.325 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:23.325 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:23.325 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:23.325 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:23.326 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:23.326 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:23.326 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:23.327 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:23.404 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:23.404 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:23.405 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:23.405 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:23.405 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:23.405 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:23.405 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:23.405 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:23.406 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:23.407 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=243), Traces: [0ms] refreshing cache from master, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [3ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [21ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [23ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [24ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [41ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [43ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [61ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [63ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [81ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [83ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [101ms] refreshing cache from master, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [103ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [161ms] refreshing cache from master, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [162ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [162ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [162ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [163ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [164ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [164ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [241ms] refreshing cache from master, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [242ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [242ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [243ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [243ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, deferred=Deferred@550410895(state=PENDING, result=null, callback=(continuation of Deferred@1642825034 after retry RPC after error@982928196) -> (continuation of Deferred@184700094 after retry RPC after error@1598858675) -> (continuation of Deferred@1647356613 after retry RPC after error@1785987500) -> (continuation of Deferred@1938474976 after retry RPC after error@1230083359) -> (continuation of Deferred@785410162 after retry RPC after error@1692628325) -> (continuation of Deferred@1098058718 after retry RPC after error@80713602) -> (continuation of Deferred@703724689 after retry RPC after error@1455881367), errback=(continuation of Deferred@1642825034 after retry RPC after error@982928196) -> (continuation of Deferred@184700094 after retry RPC after error@1598858675) -> (continuation of Deferred@1647356613 after retry RPC after error@1785987500) -> (continuation of Deferred@1938474976 after retry RPC after error@1230083359) -> (continuation of Deferred@785410162 after retry RPC after error@1692628325) -> (continuation of Deferred@1098058718 after retry RPC after error@80713602) -> (continuation of Deferred@703724689 after retry RPC after error@1455881367)))
21:04:23.407 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=243), Traces: [0ms] refreshing cache from master, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [3ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [21ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [23ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [24ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [41ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [43ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [61ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [63ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [81ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [83ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [101ms] refreshing cache from master, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [103ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [161ms] refreshing cache from master, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [162ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [162ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [162ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [163ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [164ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [164ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [241ms] refreshing cache from master, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [242ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [242ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [243ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [243ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, deferred=Deferred@550410895(state=PENDING, result=null, callback=(continuation of Deferred@1642825034 after retry RPC after error@982928196) -> (continuation of Deferred@184700094 after retry RPC after error@1598858675) -> (continuation of Deferred@1647356613 after retry RPC after error@1785987500) -> (continuation of Deferred@1938474976 after retry RPC after error@1230083359) -> (continuation of Deferred@785410162 after retry RPC after error@1692628325) -> (continuation of Deferred@1098058718 after retry RPC after error@80713602) -> (continuation of Deferred@703724689 after retry RPC after error@1455881367), errback=(continuation of Deferred@1642825034 after retry RPC after error@982928196) -> (continuation of Deferred@184700094 after retry RPC after error@1598858675) -> (continuation of Deferred@1647356613 after retry RPC after error@1785987500) -> (continuation of Deferred@1938474976 after retry RPC after error@1230083359) -> (continuation of Deferred@785410162 after retry RPC after error@1692628325) -> (continuation of Deferred@1098058718 after retry RPC after error@80713602) -> (continuation of Deferred@703724689 after retry RPC after error@1455881367)))
21:04:23.424 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:23.425 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:23.425 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:23.425 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:23.425 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:23.425 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:23.426 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:23.426 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:23.426 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:23.444 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:23.445 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:23.446 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:23.446 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:23.446 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:23.446 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:23.446 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:23.446 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:23.446 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:23.446 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:23.447 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:23.447 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:23.465 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:23.465 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:23.466 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:23.466 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:23.466 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:23.467 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:23.467 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:23.467 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:23.467 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:23.484 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:23.484 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:23.485 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:23.485 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:23.485 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:23.485 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:23.485 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:23.485 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:23.486 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:23.504 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:23.504 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:23.505 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:23.505 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:23.505 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:23.505 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:23.505 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:23.505 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:23.506 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:23.524 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:23.525 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:23.526 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:23.527 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:23.527 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:23.527 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:23.527 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:23.527 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:23.527 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:23.604 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:23.604 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:23.605 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:23.605 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:23.606 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:23.606 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:23.606 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:23.606 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:23.606 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:23.644 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:23.645 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:23.645 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:23.645 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:23.645 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:23.645 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:23.645 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:23.646 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:23.646 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:23.664 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:23.665 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:23.665 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:23.665 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:23.666 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:23.666 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:23.666 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:23.666 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:23.667 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:23.667 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=9, TimeoutTracker(timeout=250, elapsed=242), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [21ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [23ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [43ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [43ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [43ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [103ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [103ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [103ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [103ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [182ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [182ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [182ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [182ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [221ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [221ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [222ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [222ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [241ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [242ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [242ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [243ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, deferred=Deferred@945430010(state=PENDING, result=null, callback=(continuation of Deferred@210104207 after retry RPC after error@236140268) -> (continuation of Deferred@143844352 after retry RPC after error@1615830309) -> (continuation of Deferred@1125515060 after retry RPC after error@1411782124) -> (continuation of Deferred@1259030924 after retry RPC after error@1471234941) -> (continuation of Deferred@993127136 after retry RPC after error@1608253727) -> (continuation of Deferred@1309307969 after retry RPC after error@335379542) -> (continuation of Deferred@665842211 after retry RPC after error@407165744) -> (continuation of Deferred@99683333 after retry RPC after error@2098650913), errback=(continuation of Deferred@210104207 after retry RPC after error@236140268) -> (continuation of Deferred@143844352 after retry RPC after error@1615830309) -> (continuation of Deferred@1125515060 after retry RPC after error@1411782124) -> (continuation of Deferred@1259030924 after retry RPC after error@1471234941) -> (continuation of Deferred@993127136 after retry RPC after error@1608253727) -> (continuation of Deferred@1309307969 after retry RPC after error@335379542) -> (continuation of Deferred@665842211 after retry RPC after error@407165744) -> (continuation of Deferred@99683333 after retry RPC after error@2098650913)))
21:04:23.668 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=9, TimeoutTracker(timeout=250, elapsed=242), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [21ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [23ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [43ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [43ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [43ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [103ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [103ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [103ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [103ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [182ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [182ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [182ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [182ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [221ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [221ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [222ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [222ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [241ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [242ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [242ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [243ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, deferred=Deferred@945430010(state=PENDING, result=null, callback=(continuation of Deferred@210104207 after retry RPC after error@236140268) -> (continuation of Deferred@143844352 after retry RPC after error@1615830309) -> (continuation of Deferred@1125515060 after retry RPC after error@1411782124) -> (continuation of Deferred@1259030924 after retry RPC after error@1471234941) -> (continuation of Deferred@993127136 after retry RPC after error@1608253727) -> (continuation of Deferred@1309307969 after retry RPC after error@335379542) -> (continuation of Deferred@665842211 after retry RPC after error@407165744) -> (continuation of Deferred@99683333 after retry RPC after error@2098650913), errback=(continuation of Deferred@210104207 after retry RPC after error@236140268) -> (continuation of Deferred@143844352 after retry RPC after error@1615830309) -> (continuation of Deferred@1125515060 after retry RPC after error@1411782124) -> (continuation of Deferred@1259030924 after retry RPC after error@1471234941) -> (continuation of Deferred@993127136 after retry RPC after error@1608253727) -> (continuation of Deferred@1309307969 after retry RPC after error@335379542) -> (continuation of Deferred@665842211 after retry RPC after error@407165744) -> (continuation of Deferred@99683333 after retry RPC after error@2098650913)))
21:04:23.684 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:23.685 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:23.685 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:23.685 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:23.685 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:23.686 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:23.686 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:23.686 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:23.687 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:23.704 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:23.705 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:23.705 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:23.705 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:23.705 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:23.705 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:23.706 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:23.706 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:23.707 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:23.724 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:23.725 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:23.726 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:23.726 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:23.726 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:23.726 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:23.726 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:23.726 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:23.727 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:23.745 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:23.745 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:23.745 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:23.745 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:23.746 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:23.746 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:23.746 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:23.746 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:23.747 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:23.764 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:23.765 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:23.766 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:23.766 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:23.766 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:23.766 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:23.766 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:23.766 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:23.767 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:23.785 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:23.785 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:23.785 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:23.786 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:23.786 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:23.786 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:23.786 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:23.786 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:23.787 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:23.825 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:23.825 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:23.826 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:23.826 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:23.826 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:23.826 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:23.826 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:23.826 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:23.827 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:23.844 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:23.844 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:23.845 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:23.845 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:23.845 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:23.845 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:23.845 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:23.846 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:23.846 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:23.864 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:23.865 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:23.865 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:23.865 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:23.866 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:23.866 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:23.866 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:23.866 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:23.867 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:23.867 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=182), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [3ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [63ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [83ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [103ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [142ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [142ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [143ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [143ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [181ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [182ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [182ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [183ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, deferred=Deferred@461719799(state=PENDING, result=null, callback=(continuation of Deferred@767647360 after retry RPC after error@1656580524) -> (continuation of Deferred@1183882445 after retry RPC after error@2003019288) -> (continuation of Deferred@1358021418 after retry RPC after error@1493704849) -> (continuation of Deferred@465353097 after retry RPC after error@1904578432) -> (continuation of Deferred@791935497 after retry RPC after error@933714902) -> (continuation of Deferred@1227476058 after retry RPC after error@344830832) -> (continuation of Deferred@618586202 after retry RPC after error@690644732), errback=(continuation of Deferred@767647360 after retry RPC after error@1656580524) -> (continuation of Deferred@1183882445 after retry RPC after error@2003019288) -> (continuation of Deferred@1358021418 after retry RPC after error@1493704849) -> (continuation of Deferred@465353097 after retry RPC after error@1904578432) -> (continuation of Deferred@791935497 after retry RPC after error@933714902) -> (continuation of Deferred@1227476058 after retry RPC after error@344830832) -> (continuation of Deferred@618586202 after retry RPC after error@690644732)))
21:04:23.868 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=182), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [3ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [63ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [83ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [103ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [142ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [142ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [143ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [143ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [181ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [182ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [182ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [183ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, deferred=Deferred@461719799(state=PENDING, result=null, callback=(continuation of Deferred@767647360 after retry RPC after error@1656580524) -> (continuation of Deferred@1183882445 after retry RPC after error@2003019288) -> (continuation of Deferred@1358021418 after retry RPC after error@1493704849) -> (continuation of Deferred@465353097 after retry RPC after error@1904578432) -> (continuation of Deferred@791935497 after retry RPC after error@933714902) -> (continuation of Deferred@1227476058 after retry RPC after error@344830832) -> (continuation of Deferred@618586202 after retry RPC after error@690644732), errback=(continuation of Deferred@767647360 after retry RPC after error@1656580524) -> (continuation of Deferred@1183882445 after retry RPC after error@2003019288) -> (continuation of Deferred@1358021418 after retry RPC after error@1493704849) -> (continuation of Deferred@465353097 after retry RPC after error@1904578432) -> (continuation of Deferred@791935497 after retry RPC after error@933714902) -> (continuation of Deferred@1227476058 after retry RPC after error@344830832) -> (continuation of Deferred@618586202 after retry RPC after error@690644732)))
21:04:23.885 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:23.885 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:23.886 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:23.886 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:23.886 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:23.886 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:23.886 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:23.886 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:23.887 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:23.905 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:23.905 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:23.905 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:23.906 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:23.906 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:23.906 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:23.906 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:23.906 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:23.907 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:23.925 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:23.925 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:23.926 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:23.926 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:23.926 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:23.926 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:23.926 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:23.926 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:23.927 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:23.944 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:23.944 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:23.945 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:23.945 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:23.945 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:23.945 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:23.945 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:23.945 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:23.946 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:23.964 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:23.964 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:23.965 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:23.965 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:23.965 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:23.965 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:23.965 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:23.965 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:23.966 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:24.004 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:24.004 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:24.005 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:24.005 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:24.005 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:24.005 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:24.005 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:24.005 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:24.006 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:24.085 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:24.085 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:24.086 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:24.086 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:24.086 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:24.086 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:24.086 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:24.086 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:24.087 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:24.124 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:24.124 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:24.125 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:24.125 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:24.125 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:24.125 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:24.125 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:24.125 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:24.126 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:24.127 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=241), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [3ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [23ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [43ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [59ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [79ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [200ms] refreshing cache from master, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [202ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [202ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [203ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [203ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [241ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [241ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [242ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [242ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, deferred=Deferred@1256537284(state=PENDING, result=null, callback=(continuation of Deferred@489180536 after retry RPC after error@1020627367) -> (continuation of Deferred@835156346 after retry RPC after error@1386705050) -> (continuation of Deferred@222012475 after retry RPC after error@489917578) -> (continuation of Deferred@397753087 after retry RPC after error@2122300211) -> (continuation of Deferred@783663803 after retry RPC after error@803425339) -> (continuation of Deferred@289401338 after retry RPC after error@312114893) -> (continuation of Deferred@314234431 after retry RPC after error@655213255), errback=(continuation of Deferred@489180536 after retry RPC after error@1020627367) -> (continuation of Deferred@835156346 after retry RPC after error@1386705050) -> (continuation of Deferred@222012475 after retry RPC after error@489917578) -> (continuation of Deferred@397753087 after retry RPC after error@2122300211) -> (continuation of Deferred@783663803 after retry RPC after error@803425339) -> (continuation of Deferred@289401338 after retry RPC after error@312114893) -> (continuation of Deferred@314234431 after retry RPC after error@655213255)))
21:04:24.127 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=241), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [3ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [23ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [43ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [59ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [79ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [200ms] refreshing cache from master, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [202ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [202ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [203ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [203ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [241ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [241ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [242ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [242ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, deferred=Deferred@1256537284(state=PENDING, result=null, callback=(continuation of Deferred@489180536 after retry RPC after error@1020627367) -> (continuation of Deferred@835156346 after retry RPC after error@1386705050) -> (continuation of Deferred@222012475 after retry RPC after error@489917578) -> (continuation of Deferred@397753087 after retry RPC after error@2122300211) -> (continuation of Deferred@783663803 after retry RPC after error@803425339) -> (continuation of Deferred@289401338 after retry RPC after error@312114893) -> (continuation of Deferred@314234431 after retry RPC after error@655213255), errback=(continuation of Deferred@489180536 after retry RPC after error@1020627367) -> (continuation of Deferred@835156346 after retry RPC after error@1386705050) -> (continuation of Deferred@222012475 after retry RPC after error@489917578) -> (continuation of Deferred@397753087 after retry RPC after error@2122300211) -> (continuation of Deferred@783663803 after retry RPC after error@803425339) -> (continuation of Deferred@289401338 after retry RPC after error@312114893) -> (continuation of Deferred@314234431 after retry RPC after error@655213255)))
21:04:24.144 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:24.144 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:24.145 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:24.145 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:24.145 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:24.145 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:24.145 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:24.145 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:24.146 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:24.164 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:24.164 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:24.165 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:24.165 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:24.165 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:24.165 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:24.165 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:24.165 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:24.166 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:24.184 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:24.184 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:24.185 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:24.185 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:24.185 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:24.185 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:24.185 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:24.185 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:24.186 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:24.204 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:24.204 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:24.205 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:24.205 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:24.205 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:24.205 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:24.205 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:24.205 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:24.206 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:24.224 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:24.224 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:24.225 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:24.225 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:24.225 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:24.225 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:24.225 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:24.226 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:24.226 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:24.264 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:24.265 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:24.265 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:24.265 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:24.266 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:24.266 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:24.266 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:24.266 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:24.267 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:24.304 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:24.305 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:24.305 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:24.305 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:24.305 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:24.306 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:24.306 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:24.306 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:24.307 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:24.307 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=163), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [39ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [162ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [162ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [163ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [163ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, deferred=Deferred@2054336643(state=PENDING, result=null, callback=(continuation of Deferred@289692858 after retry RPC after error@473723849) -> (continuation of Deferred@930139817 after retry RPC after error@1483310700) -> (continuation of Deferred@1803264087 after retry RPC after error@338044106) -> (continuation of Deferred@621978977 after retry RPC after error@828120393) -> (continuation of Deferred@118716973 after retry RPC after error@573967985) -> (continuation of Deferred@314547963 after retry RPC after error@1138257840), errback=(continuation of Deferred@289692858 after retry RPC after error@473723849) -> (continuation of Deferred@930139817 after retry RPC after error@1483310700) -> (continuation of Deferred@1803264087 after retry RPC after error@338044106) -> (continuation of Deferred@621978977 after retry RPC after error@828120393) -> (continuation of Deferred@118716973 after retry RPC after error@573967985) -> (continuation of Deferred@314547963 after retry RPC after error@1138257840)))
21:04:24.308 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=163), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [39ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [162ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [162ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [163ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [163ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, deferred=Deferred@2054336643(state=PENDING, result=null, callback=(continuation of Deferred@289692858 after retry RPC after error@473723849) -> (continuation of Deferred@930139817 after retry RPC after error@1483310700) -> (continuation of Deferred@1803264087 after retry RPC after error@338044106) -> (continuation of Deferred@621978977 after retry RPC after error@828120393) -> (continuation of Deferred@118716973 after retry RPC after error@573967985) -> (continuation of Deferred@314547963 after retry RPC after error@1138257840), errback=(continuation of Deferred@289692858 after retry RPC after error@473723849) -> (continuation of Deferred@930139817 after retry RPC after error@1483310700) -> (continuation of Deferred@1803264087 after retry RPC after error@338044106) -> (continuation of Deferred@621978977 after retry RPC after error@828120393) -> (continuation of Deferred@118716973 after retry RPC after error@573967985) -> (continuation of Deferred@314547963 after retry RPC after error@1138257840)))
21:04:24.324 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:24.325 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:24.325 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:24.325 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:24.326 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:24.326 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:24.326 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:24.326 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:24.327 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:24.344 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:24.345 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:24.345 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:24.345 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:24.346 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:24.346 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:24.346 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:24.346 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:24.347 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:24.365 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:24.365 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:24.365 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:24.366 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:24.366 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:24.366 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:24.366 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:24.366 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:24.367 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:24.385 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:24.385 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:24.386 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:24.386 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:24.386 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:24.386 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:24.386 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:24.386 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:24.387 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:24.405 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:24.405 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:24.406 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:24.406 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:24.406 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:24.406 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:24.406 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:24.406 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:24.407 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:24.444 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:24.444 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:24.445 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:24.445 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:24.445 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:24.445 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:24.445 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:24.445 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:24.446 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:24.484 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:24.485 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:24.485 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:24.485 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:24.485 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:24.486 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:24.486 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:24.486 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:24.487 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:24.487 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=162), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [23ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [43ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [63ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [83ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [161ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [162ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [163ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [163ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, deferred=Deferred@96080598(state=PENDING, result=null, callback=(continuation of Deferred@1913996570 after retry RPC after error@638721817) -> (continuation of Deferred@1116441497 after retry RPC after error@393354684) -> (continuation of Deferred@1326954461 after retry RPC after error@2034992113) -> (continuation of Deferred@1728186370 after retry RPC after error@1612342947) -> (continuation of Deferred@306285602 after retry RPC after error@934334864) -> (continuation of Deferred@1082498428 after retry RPC after error@1939461297), errback=(continuation of Deferred@1913996570 after retry RPC after error@638721817) -> (continuation of Deferred@1116441497 after retry RPC after error@393354684) -> (continuation of Deferred@1326954461 after retry RPC after error@2034992113) -> (continuation of Deferred@1728186370 after retry RPC after error@1612342947) -> (continuation of Deferred@306285602 after retry RPC after error@934334864) -> (continuation of Deferred@1082498428 after retry RPC after error@1939461297)))
21:04:24.488 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=162), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [23ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [43ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [63ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [83ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [161ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [162ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [163ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [163ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, deferred=Deferred@96080598(state=PENDING, result=null, callback=(continuation of Deferred@1913996570 after retry RPC after error@638721817) -> (continuation of Deferred@1116441497 after retry RPC after error@393354684) -> (continuation of Deferred@1326954461 after retry RPC after error@2034992113) -> (continuation of Deferred@1728186370 after retry RPC after error@1612342947) -> (continuation of Deferred@306285602 after retry RPC after error@934334864) -> (continuation of Deferred@1082498428 after retry RPC after error@1939461297), errback=(continuation of Deferred@1913996570 after retry RPC after error@638721817) -> (continuation of Deferred@1116441497 after retry RPC after error@393354684) -> (continuation of Deferred@1326954461 after retry RPC after error@2034992113) -> (continuation of Deferred@1728186370 after retry RPC after error@1612342947) -> (continuation of Deferred@306285602 after retry RPC after error@934334864) -> (continuation of Deferred@1082498428 after retry RPC after error@1939461297)))
21:04:24.504 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:24.505 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:24.505 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:24.505 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:24.506 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:24.506 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:24.506 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:24.506 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:24.507 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:24.524 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:24.525 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:24.525 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:24.525 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:24.526 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:24.526 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:24.526 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:24.526 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:24.526 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:24.544 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:24.545 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:24.545 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:24.546 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:24.546 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:24.546 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:24.546 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:24.546 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:24.547 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:24.565 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:24.565 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:24.566 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:24.566 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:24.566 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:24.566 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:24.566 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:24.566 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:24.567 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:24.585 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:24.585 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:24.586 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:24.586 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:24.586 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:24.586 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:24.586 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:24.586 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:24.587 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:24.605 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:24.605 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:24.606 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:24.606 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:24.606 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:24.606 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:24.606 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:24.606 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:24.607 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:24.625 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:24.625 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:24.626 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:24.626 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:24.626 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:24.626 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:24.626 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:24.626 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:24.627 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:24.744 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:24.745 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:24.745 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:24.745 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:24.746 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:24.746 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:24.746 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:24.746 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:24.747 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:24.747 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=242), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [43ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [63ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [83ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [103ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [120ms] refreshing cache from master, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [123ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [241ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [242ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [242ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [243ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, deferred=Deferred@1153080106(state=PENDING, result=null, callback=(continuation of Deferred@1091891836 after retry RPC after error@410127337) -> (continuation of Deferred@1818998513 after retry RPC after error@2090341277) -> (continuation of Deferred@452565344 after retry RPC after error@316470421) -> (continuation of Deferred@590418945 after retry RPC after error@520744705) -> (continuation of Deferred@1240287001 after retry RPC after error@1012955885) -> (continuation of Deferred@391835772 after retry RPC after error@1199273073) -> (continuation of Deferred@1678587400 after retry RPC after error@1535483766), errback=(continuation of Deferred@1091891836 after retry RPC after error@410127337) -> (continuation of Deferred@1818998513 after retry RPC after error@2090341277) -> (continuation of Deferred@452565344 after retry RPC after error@316470421) -> (continuation of Deferred@590418945 after retry RPC after error@520744705) -> (continuation of Deferred@1240287001 after retry RPC after error@1012955885) -> (continuation of Deferred@391835772 after retry RPC after error@1199273073) -> (continuation of Deferred@1678587400 after retry RPC after error@1535483766)))
21:04:24.748 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=242), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [43ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [63ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [83ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [103ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [120ms] refreshing cache from master, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [123ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [241ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [242ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [242ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [243ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, deferred=Deferred@1153080106(state=PENDING, result=null, callback=(continuation of Deferred@1091891836 after retry RPC after error@410127337) -> (continuation of Deferred@1818998513 after retry RPC after error@2090341277) -> (continuation of Deferred@452565344 after retry RPC after error@316470421) -> (continuation of Deferred@590418945 after retry RPC after error@520744705) -> (continuation of Deferred@1240287001 after retry RPC after error@1012955885) -> (continuation of Deferred@391835772 after retry RPC after error@1199273073) -> (continuation of Deferred@1678587400 after retry RPC after error@1535483766), errback=(continuation of Deferred@1091891836 after retry RPC after error@410127337) -> (continuation of Deferred@1818998513 after retry RPC after error@2090341277) -> (continuation of Deferred@452565344 after retry RPC after error@316470421) -> (continuation of Deferred@590418945 after retry RPC after error@520744705) -> (continuation of Deferred@1240287001 after retry RPC after error@1012955885) -> (continuation of Deferred@391835772 after retry RPC after error@1199273073) -> (continuation of Deferred@1678587400 after retry RPC after error@1535483766)))
21:04:24.764 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:24.765 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:24.765 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:24.766 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:24.766 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:24.766 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:24.766 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:24.766 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:24.767 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:24.785 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:24.785 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:24.785 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:24.786 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:24.786 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:24.786 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:24.786 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:24.786 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:24.787 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:24.805 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:24.805 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:24.806 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:24.806 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:24.806 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:24.806 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:24.806 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:24.807 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:24.807 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:24.824 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:24.824 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:24.825 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:24.825 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:24.825 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:24.826 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:24.826 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:24.826 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:24.826 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:24.844 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:24.844 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:24.845 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:24.845 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:24.845 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:24.845 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:24.845 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:24.845 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:24.846 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:24.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:24.850308 24283 raft_consensus.cc:493] T 00000000000000000000000000000000 P eb664add87f44013b596f4cc01c9a41c [term 4 FOLLOWER]: Starting pre-election (detected failure of leader c520077acf3341e29447c6c2b8d3f74c)
21:04:24.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:24.850569 24283 raft_consensus.cc:515] T 00000000000000000000000000000000 P eb664add87f44013b596f4cc01c9a41c [term 4 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e9dd218ca2244f5a934aac2b7e58eb1d" member_type: VOTER last_known_addr { host: "127.22.168.62" port: 37239 } } peers { permanent_uuid: "c520077acf3341e29447c6c2b8d3f74c" member_type: VOTER last_known_addr { host: "127.22.168.61" port: 32947 } } peers { permanent_uuid: "eb664add87f44013b596f4cc01c9a41c" member_type: VOTER last_known_addr { host: "127.22.168.60" port: 33247 } }
21:04:24.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:24.851317 24283 leader_election.cc:290] T 00000000000000000000000000000000 P eb664add87f44013b596f4cc01c9a41c [CANDIDATE]: Term 5 pre-election: Requested pre-vote from peers e9dd218ca2244f5a934aac2b7e58eb1d (127.22.168.62:37239), c520077acf3341e29447c6c2b8d3f74c (127.22.168.61:32947)
21:04:24.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:24.853773 24086 leader_election.cc:336] T 00000000000000000000000000000000 P eb664add87f44013b596f4cc01c9a41c [CANDIDATE]: Term 5 pre-election: RPC error from VoteRequest() call to peer e9dd218ca2244f5a934aac2b7e58eb1d (127.22.168.62:37239): Network error: Client connection negotiation failed: client connection to 127.22.168.62:37239: connect: Connection refused (error 111)
21:04:24.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:24.853963 24089 leader_election.cc:336] T 00000000000000000000000000000000 P eb664add87f44013b596f4cc01c9a41c [CANDIDATE]: Term 5 pre-election: RPC error from VoteRequest() call to peer c520077acf3341e29447c6c2b8d3f74c (127.22.168.61:32947): Network error: Client connection negotiation failed: client connection to 127.22.168.61:32947: connect: Connection refused (error 111)
21:04:24.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:24.854125 24089 leader_election.cc:304] T 00000000000000000000000000000000 P eb664add87f44013b596f4cc01c9a41c [CANDIDATE]: Term 5 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: eb664add87f44013b596f4cc01c9a41c; no voters: c520077acf3341e29447c6c2b8d3f74c, e9dd218ca2244f5a934aac2b7e58eb1d
21:04:24.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:24.854461 24283 raft_consensus.cc:2749] T 00000000000000000000000000000000 P eb664add87f44013b596f4cc01c9a41c [term 4 FOLLOWER]: Leader pre-election lost for term 5. Reason: could not achieve majority
21:04:24.884 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:24.885 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:24.885 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:24.885 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:24.885 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:24.885 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:24.885 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:24.886 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:24.886 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:24.904 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:24.905 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:24.905 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:24.905 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:24.905 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:24.905 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:24.905 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:24.905 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:24.906 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:24.907 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=142), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [3ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [40ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [43ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [59ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [141ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [141ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [142ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [143ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, deferred=Deferred@1586403880(state=PENDING, result=null, callback=(continuation of Deferred@417510526 after retry RPC after error@1467058215) -> (continuation of Deferred@464622460 after retry RPC after error@834934264) -> (continuation of Deferred@1007163130 after retry RPC after error@1742963180) -> (continuation of Deferred@1885262879 after retry RPC after error@1629779001) -> (continuation of Deferred@493293857 after retry RPC after error@1727012736) -> (continuation of Deferred@1611186541 after retry RPC after error@575665385), errback=(continuation of Deferred@417510526 after retry RPC after error@1467058215) -> (continuation of Deferred@464622460 after retry RPC after error@834934264) -> (continuation of Deferred@1007163130 after retry RPC after error@1742963180) -> (continuation of Deferred@1885262879 after retry RPC after error@1629779001) -> (continuation of Deferred@493293857 after retry RPC after error@1727012736) -> (continuation of Deferred@1611186541 after retry RPC after error@575665385)))
21:04:24.908 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=142), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [3ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [40ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [43ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [59ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [141ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [141ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [142ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [143ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, deferred=Deferred@1586403880(state=PENDING, result=null, callback=(continuation of Deferred@417510526 after retry RPC after error@1467058215) -> (continuation of Deferred@464622460 after retry RPC after error@834934264) -> (continuation of Deferred@1007163130 after retry RPC after error@1742963180) -> (continuation of Deferred@1885262879 after retry RPC after error@1629779001) -> (continuation of Deferred@493293857 after retry RPC after error@1727012736) -> (continuation of Deferred@1611186541 after retry RPC after error@575665385), errback=(continuation of Deferred@417510526 after retry RPC after error@1467058215) -> (continuation of Deferred@464622460 after retry RPC after error@834934264) -> (continuation of Deferred@1007163130 after retry RPC after error@1742963180) -> (continuation of Deferred@1885262879 after retry RPC after error@1629779001) -> (continuation of Deferred@493293857 after retry RPC after error@1727012736) -> (continuation of Deferred@1611186541 after retry RPC after error@575665385)))
21:04:24.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:24.913165 24071 master_runner.cc:186] Error getting master registration for 127.22.168.62:37239: Network error: Client connection negotiation failed: client connection to 127.22.168.62:37239: connect: Connection refused (error 111),
21:04:24.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:24.913371 24071 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
21:04:24.924 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:24.925 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:24.925 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:24.925 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:24.925 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:24.925 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:24.925 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:24.925 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:24.926 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:24.944 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:24.945 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:24.945 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:24.945 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:24.946 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:24.946 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:24.946 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:24.946 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:24.947 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:24.964 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:24.965 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:24.965 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:24.965 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:24.966 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:24.966 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:24.966 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:24.966 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:24.967 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:24.984 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:24.985 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:24.985 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:24.985 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:24.985 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:24.986 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:24.986 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:24.986 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:24.987 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:25.004 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:25.005 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:25.005 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:25.005 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:25.006 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:25.006 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:25.006 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:25.006 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:25.006 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:25.045 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:25.045 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:25.046 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:25.046 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:25.046 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:25.046 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:25.046 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:25.046 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:25.047 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:25.065 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:25.065 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:25.066 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:25.066 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:25.066 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:25.066 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:25.066 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:25.066 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:25.067 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:25.068 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=143), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [43ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [63ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [123ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [142ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [142ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [143ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [143ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, deferred=Deferred@1526903272(state=PENDING, result=null, callback=(continuation of Deferred@351565870 after retry RPC after error@673249455) -> (continuation of Deferred@815685312 after retry RPC after error@2128422090) -> (continuation of Deferred@1230964279 after retry RPC after error@1712679475) -> (continuation of Deferred@616513647 after retry RPC after error@861129542) -> (continuation of Deferred@42737061 after retry RPC after error@1334164277) -> (continuation of Deferred@300361362 after retry RPC after error@1011078675), errback=(continuation of Deferred@351565870 after retry RPC after error@673249455) -> (continuation of Deferred@815685312 after retry RPC after error@2128422090) -> (continuation of Deferred@1230964279 after retry RPC after error@1712679475) -> (continuation of Deferred@616513647 after retry RPC after error@861129542) -> (continuation of Deferred@42737061 after retry RPC after error@1334164277) -> (continuation of Deferred@300361362 after retry RPC after error@1011078675)))
21:04:25.069 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=143), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [43ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [63ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [123ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [142ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [142ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [143ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [143ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, deferred=Deferred@1526903272(state=PENDING, result=null, callback=(continuation of Deferred@351565870 after retry RPC after error@673249455) -> (continuation of Deferred@815685312 after retry RPC after error@2128422090) -> (continuation of Deferred@1230964279 after retry RPC after error@1712679475) -> (continuation of Deferred@616513647 after retry RPC after error@861129542) -> (continuation of Deferred@42737061 after retry RPC after error@1334164277) -> (continuation of Deferred@300361362 after retry RPC after error@1011078675), errback=(continuation of Deferred@351565870 after retry RPC after error@673249455) -> (continuation of Deferred@815685312 after retry RPC after error@2128422090) -> (continuation of Deferred@1230964279 after retry RPC after error@1712679475) -> (continuation of Deferred@616513647 after retry RPC after error@861129542) -> (continuation of Deferred@42737061 after retry RPC after error@1334164277) -> (continuation of Deferred@300361362 after retry RPC after error@1011078675)))
21:04:25.085 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:25.085 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:25.086 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:25.086 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:25.086 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:25.086 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:25.086 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:25.086 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:25.087 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:25.104 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:25.105 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:25.106 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:25.106 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:25.106 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:25.106 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:25.106 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:25.106 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:25.107 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:25.124 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:25.125 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:25.126 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:25.126 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:25.126 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:25.126 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:25.126 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:25.126 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:25.127 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:25.144 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:25.145 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:25.145 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:25.145 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:25.146 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:25.146 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:25.146 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:25.146 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:25.148 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:25.165 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:25.165 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:25.166 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:25.166 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:25.166 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:25.166 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:25.166 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:25.166 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:25.167 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:25.204 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:25.204 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:25.205 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:25.205 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:25.205 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:25.205 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:25.205 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:25.205 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:25.206 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:25.224 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:25.224 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:25.225 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:25.225 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:25.225 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:25.225 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:25.225 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:25.225 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:25.227 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:25.284 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:25.285 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:25.286 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:25.286 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:25.286 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:25.287 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:25.287 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:25.287 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:25.288 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:25.289 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=203), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [23ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [64ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [64ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [83ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [119ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [141ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [141ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [143ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [143ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [202ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [203ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [204ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [204ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, deferred=Deferred@338798563(state=PENDING, result=null, callback=(continuation of Deferred@1258436205 after retry RPC after error@2000446370) -> (continuation of Deferred@437949150 after retry RPC after error@51192001) -> (continuation of Deferred@1499910035 after retry RPC after error@1297319534) -> (continuation of Deferred@51474808 after retry RPC after error@1768968947) -> (continuation of Deferred@109384217 after retry RPC after error@1471452241) -> (continuation of Deferred@1604756846 after retry RPC after error@883590051) -> (continuation of Deferred@437345013 after retry RPC after error@1769231354), errback=(continuation of Deferred@1258436205 after retry RPC after error@2000446370) -> (continuation of Deferred@437949150 after retry RPC after error@51192001) -> (continuation of Deferred@1499910035 after retry RPC after error@1297319534) -> (continuation of Deferred@51474808 after retry RPC after error@1768968947) -> (continuation of Deferred@109384217 after retry RPC after error@1471452241) -> (continuation of Deferred@1604756846 after retry RPC after error@883590051) -> (continuation of Deferred@437345013 after retry RPC after error@1769231354)))
21:04:25.289 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=203), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [23ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [64ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [64ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [83ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [119ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [141ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [141ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [143ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [143ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [202ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [203ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [204ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [204ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, deferred=Deferred@338798563(state=PENDING, result=null, callback=(continuation of Deferred@1258436205 after retry RPC after error@2000446370) -> (continuation of Deferred@437949150 after retry RPC after error@51192001) -> (continuation of Deferred@1499910035 after retry RPC after error@1297319534) -> (continuation of Deferred@51474808 after retry RPC after error@1768968947) -> (continuation of Deferred@109384217 after retry RPC after error@1471452241) -> (continuation of Deferred@1604756846 after retry RPC after error@883590051) -> (continuation of Deferred@437345013 after retry RPC after error@1769231354), errback=(continuation of Deferred@1258436205 after retry RPC after error@2000446370) -> (continuation of Deferred@437949150 after retry RPC after error@51192001) -> (continuation of Deferred@1499910035 after retry RPC after error@1297319534) -> (continuation of Deferred@51474808 after retry RPC after error@1768968947) -> (continuation of Deferred@109384217 after retry RPC after error@1471452241) -> (continuation of Deferred@1604756846 after retry RPC after error@883590051) -> (continuation of Deferred@437345013 after retry RPC after error@1769231354)))
21:04:25.304 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:25.305 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:25.305 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:25.305 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:25.305 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:25.306 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:25.306 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:25.306 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:25.307 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:25.324 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:25.325 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:25.325 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:25.325 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:25.325 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:25.326 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:25.326 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:25.326 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:25.327 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:25.344 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:25.345 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:25.345 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:25.345 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:25.345 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:25.345 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:25.346 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:25.346 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:25.347 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:25.364 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:25.365 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:25.365 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:25.365 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:25.366 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:25.366 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:25.366 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:25.366 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:25.367 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:25.384 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:25.385 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:25.385 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:25.385 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:25.385 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:25.385 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:25.386 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:25.386 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:25.387 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:25.424 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:25.425 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:25.425 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:25.425 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:25.425 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:25.426 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:25.426 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:25.426 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:25.427 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:25.465 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:25.465 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:25.465 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:25.466 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:25.466 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:25.466 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:25.466 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:25.466 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:25.467 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:25.484 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:25.485 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:25.485 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:25.485 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:25.485 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:25.486 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:25.486 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:25.486 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:25.487 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:25.544 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:25.544 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:25.545 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:25.545 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:25.545 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:25.545 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:25.545 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:25.545 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:25.546 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:25.552 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=9, TimeoutTracker(timeout=250, elapsed=242), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [23ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [43ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [162ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [162ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [163ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [163ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [181ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [182ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [182ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [183ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [239ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [241ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [241ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [242ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [242ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, deferred=Deferred@1900713457(state=PENDING, result=null, callback=(continuation of Deferred@1987856513 after retry RPC after error@1813723295) -> (continuation of Deferred@2019478305 after retry RPC after error@1562262726) -> (continuation of Deferred@1236426660 after retry RPC after error@2086160845) -> (continuation of Deferred@1926333092 after retry RPC after error@1577412660) -> (continuation of Deferred@93405999 after retry RPC after error@993621105) -> (continuation of Deferred@1166214416 after retry RPC after error@808620380) -> (continuation of Deferred@1060447310 after retry RPC after error@1340036389) -> (continuation of Deferred@313090635 after retry RPC after error@917874995), errback=(continuation of Deferred@1987856513 after retry RPC after error@1813723295) -> (continuation of Deferred@2019478305 after retry RPC after error@1562262726) -> (continuation of Deferred@1236426660 after retry RPC after error@2086160845) -> (continuation of Deferred@1926333092 after retry RPC after error@1577412660) -> (continuation of Deferred@93405999 after retry RPC after error@993621105) -> (continuation of Deferred@1166214416 after retry RPC after error@808620380) -> (continuation of Deferred@1060447310 after retry RPC after error@1340036389) -> (continuation of Deferred@313090635 after retry RPC after error@917874995)))
21:04:25.553 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=9, TimeoutTracker(timeout=250, elapsed=242), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [23ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [43ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [162ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [162ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [163ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [163ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [181ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [182ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [182ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [183ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [239ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [241ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [241ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [242ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [242ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, deferred=Deferred@1900713457(state=PENDING, result=null, callback=(continuation of Deferred@1987856513 after retry RPC after error@1813723295) -> (continuation of Deferred@2019478305 after retry RPC after error@1562262726) -> (continuation of Deferred@1236426660 after retry RPC after error@2086160845) -> (continuation of Deferred@1926333092 after retry RPC after error@1577412660) -> (continuation of Deferred@93405999 after retry RPC after error@993621105) -> (continuation of Deferred@1166214416 after retry RPC after error@808620380) -> (continuation of Deferred@1060447310 after retry RPC after error@1340036389) -> (continuation of Deferred@313090635 after retry RPC after error@917874995), errback=(continuation of Deferred@1987856513 after retry RPC after error@1813723295) -> (continuation of Deferred@2019478305 after retry RPC after error@1562262726) -> (continuation of Deferred@1236426660 after retry RPC after error@2086160845) -> (continuation of Deferred@1926333092 after retry RPC after error@1577412660) -> (continuation of Deferred@93405999 after retry RPC after error@993621105) -> (continuation of Deferred@1166214416 after retry RPC after error@808620380) -> (continuation of Deferred@1060447310 after retry RPC after error@1340036389) -> (continuation of Deferred@313090635 after retry RPC after error@917874995)))
21:04:25.564 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:25.564 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:25.565 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:25.565 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:25.565 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:25.565 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:25.565 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:25.565 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:25.566 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:25.584 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:25.584 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:25.585 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:25.585 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:25.585 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:25.585 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:25.585 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:25.585 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:25.586 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:25.604 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:25.604 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:25.605 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:25.605 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:25.605 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:25.605 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:25.605 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:25.605 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:25.606 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:25.624 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:25.624 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:25.625 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:25.625 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:25.625 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:25.625 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:25.625 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:25.625 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:25.626 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:25.644 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:25.644 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:25.645 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:25.645 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:25.645 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:25.645 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:25.645 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:25.645 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:25.646 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:25.664 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:25.664 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:25.665 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:25.665 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:25.665 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:25.665 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:25.665 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:25.665 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:25.665 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:25.665 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:25.666 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:25.666 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:25.724 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:25.725 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:25.725 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:25.725 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:25.725 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:25.725 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:25.725 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:25.725 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:25.726 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:25.744 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:25.745 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:25.745 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:25.745 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:25.745 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:25.745 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:25.745 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:25.745 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:25.746 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:25.804 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:25.805 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:25.805 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:25.805 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:25.806 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:25.806 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:25.806 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:25.806 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:25.807 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:25.807 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=243), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [3ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [20ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [23ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [41ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [43ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [61ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [63ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [80ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [83ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [101ms] refreshing cache from master, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [103ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [161ms] refreshing cache from master, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [162ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [162ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [163ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [164ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [241ms] refreshing cache from master, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [242ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [242ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [243ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [243ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [244ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [244ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, deferred=Deferred@1368519322(state=PENDING, result=null, callback=(continuation of Deferred@1905896558 after retry RPC after error@122909098) -> (continuation of Deferred@854635269 after retry RPC after error@1814794979) -> (continuation of Deferred@1349364180 after retry RPC after error@1286560850) -> (continuation of Deferred@2070980185 after retry RPC after error@1266454366) -> (continuation of Deferred@1986178284 after retry RPC after error@1075389374) -> (continuation of Deferred@1042614154 after retry RPC after error@294418799) -> (continuation of Deferred@1927979646 after retry RPC after error@2070375246), errback=(continuation of Deferred@1905896558 after retry RPC after error@122909098) -> (continuation of Deferred@854635269 after retry RPC after error@1814794979) -> (continuation of Deferred@1349364180 after retry RPC after error@1286560850) -> (continuation of Deferred@2070980185 after retry RPC after error@1266454366) -> (continuation of Deferred@1986178284 after retry RPC after error@1075389374) -> (continuation of Deferred@1042614154 after retry RPC after error@294418799) -> (continuation of Deferred@1927979646 after retry RPC after error@2070375246)))
21:04:25.808 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=243), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [3ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [20ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [23ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [41ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [43ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [61ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [63ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [80ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [83ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [101ms] refreshing cache from master, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [103ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [161ms] refreshing cache from master, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [162ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [162ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [163ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [164ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [241ms] refreshing cache from master, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [242ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [242ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [243ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [243ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [244ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [244ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, deferred=Deferred@1368519322(state=PENDING, result=null, callback=(continuation of Deferred@1905896558 after retry RPC after error@122909098) -> (continuation of Deferred@854635269 after retry RPC after error@1814794979) -> (continuation of Deferred@1349364180 after retry RPC after error@1286560850) -> (continuation of Deferred@2070980185 after retry RPC after error@1266454366) -> (continuation of Deferred@1986178284 after retry RPC after error@1075389374) -> (continuation of Deferred@1042614154 after retry RPC after error@294418799) -> (continuation of Deferred@1927979646 after retry RPC after error@2070375246), errback=(continuation of Deferred@1905896558 after retry RPC after error@122909098) -> (continuation of Deferred@854635269 after retry RPC after error@1814794979) -> (continuation of Deferred@1349364180 after retry RPC after error@1286560850) -> (continuation of Deferred@2070980185 after retry RPC after error@1266454366) -> (continuation of Deferred@1986178284 after retry RPC after error@1075389374) -> (continuation of Deferred@1042614154 after retry RPC after error@294418799) -> (continuation of Deferred@1927979646 after retry RPC after error@2070375246)))
21:04:25.825 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:25.825 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:25.826 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:25.826 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:25.826 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:25.826 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:25.826 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:25.826 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:25.827 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:25.845 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:25.845 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:25.846 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:25.846 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:25.846 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:25.846 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:25.846 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:25.846 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:25.847 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:25.864 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:25.865 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:25.866 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:25.866 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:25.866 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:25.866 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:25.866 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:25.866 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:25.867 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:25.884 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:25.885 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:25.885 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:25.885 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:25.885 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:25.885 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:25.885 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:25.886 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:25.886 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:25.904 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:25.905 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:25.905 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:25.905 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:25.906 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:25.906 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:25.906 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:25.906 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:25.907 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:25.924 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:25.925 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:25.926 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:25.926 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:25.926 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:25.926 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:25.926 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:25.926 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:25.926 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:25.926 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:25.927 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:25.927 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:25.984 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:25.984 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:25.985 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:25.985 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:25.985 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:25.985 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:25.985 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:25.985 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:25.986 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:25.987 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=162), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [3ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [23ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [43ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [83ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [101ms] refreshing cache from master, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [103ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [161ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [161ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [162ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, deferred=Deferred@209975143(state=PENDING, result=null, callback=(continuation of Deferred@1285317131 after retry RPC after error@832181566) -> (continuation of Deferred@1173970861 after retry RPC after error@1456648378) -> (continuation of Deferred@2037455953 after retry RPC after error@405001636) -> (continuation of Deferred@388560186 after retry RPC after error@1413271557) -> (continuation of Deferred@635358535 after retry RPC after error@601268909) -> (continuation of Deferred@1938227912 after retry RPC after error@1091131461), errback=(continuation of Deferred@1285317131 after retry RPC after error@832181566) -> (continuation of Deferred@1173970861 after retry RPC after error@1456648378) -> (continuation of Deferred@2037455953 after retry RPC after error@405001636) -> (continuation of Deferred@388560186 after retry RPC after error@1413271557) -> (continuation of Deferred@635358535 after retry RPC after error@601268909) -> (continuation of Deferred@1938227912 after retry RPC after error@1091131461)))
21:04:25.987 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=162), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [3ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [23ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [43ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [83ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [101ms] refreshing cache from master, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [103ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [161ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [161ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [162ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, deferred=Deferred@209975143(state=PENDING, result=null, callback=(continuation of Deferred@1285317131 after retry RPC after error@832181566) -> (continuation of Deferred@1173970861 after retry RPC after error@1456648378) -> (continuation of Deferred@2037455953 after retry RPC after error@405001636) -> (continuation of Deferred@388560186 after retry RPC after error@1413271557) -> (continuation of Deferred@635358535 after retry RPC after error@601268909) -> (continuation of Deferred@1938227912 after retry RPC after error@1091131461), errback=(continuation of Deferred@1285317131 after retry RPC after error@832181566) -> (continuation of Deferred@1173970861 after retry RPC after error@1456648378) -> (continuation of Deferred@2037455953 after retry RPC after error@405001636) -> (continuation of Deferred@388560186 after retry RPC after error@1413271557) -> (continuation of Deferred@635358535 after retry RPC after error@601268909) -> (continuation of Deferred@1938227912 after retry RPC after error@1091131461)))
21:04:26.004 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:26.004 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:26.005 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:26.005 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:26.005 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:26.005 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:26.006 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:26.006 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:26.007 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:26.024 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:26.024 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:26.025 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:26.025 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:26.025 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:26.025 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:26.025 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:26.025 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:26.026 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:26.044 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:26.044 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:26.045 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:26.045 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:26.045 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:26.045 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:26.046 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:26.046 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:26.046 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:26.064 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:26.064 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:26.065 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:26.065 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:26.065 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:26.065 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:26.065 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:26.065 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:26.066 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:26.084 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:26.084 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:26.085 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:26.085 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:26.085 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:26.085 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:26.085 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:26.085 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:26.086 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:26.104 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:26.104 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:26.105 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:26.105 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:26.105 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:26.105 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:26.105 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:26.105 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:26.106 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:26.164 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:26.164 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:26.165 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:26.165 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:26.165 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:26.165 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:26.165 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:26.165 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:26.166 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:26.244 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:26.245 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:26.245 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:26.245 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:26.245 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:26.245 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:26.245 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:26.246 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:26.246 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:26.247 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=242), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [99ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [161ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [161ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [162ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [241ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [241ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [242ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [243ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, deferred=Deferred@1509020791(state=PENDING, result=null, callback=(continuation of Deferred@1665791915 after retry RPC after error@552074183) -> (continuation of Deferred@1696776920 after retry RPC after error@849866855) -> (continuation of Deferred@520076599 after retry RPC after error@22627564) -> (continuation of Deferred@1914593821 after retry RPC after error@2142736625) -> (continuation of Deferred@474123317 after retry RPC after error@1467467125) -> (continuation of Deferred@1450728204 after retry RPC after error@361457962) -> (continuation of Deferred@488664202 after retry RPC after error@1947553334), errback=(continuation of Deferred@1665791915 after retry RPC after error@552074183) -> (continuation of Deferred@1696776920 after retry RPC after error@849866855) -> (continuation of Deferred@520076599 after retry RPC after error@22627564) -> (continuation of Deferred@1914593821 after retry RPC after error@2142736625) -> (continuation of Deferred@474123317 after retry RPC after error@1467467125) -> (continuation of Deferred@1450728204 after retry RPC after error@361457962) -> (continuation of Deferred@488664202 after retry RPC after error@1947553334)))
21:04:26.247 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=242), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [99ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [161ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [161ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [162ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [241ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [241ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [242ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [243ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, deferred=Deferred@1509020791(state=PENDING, result=null, callback=(continuation of Deferred@1665791915 after retry RPC after error@552074183) -> (continuation of Deferred@1696776920 after retry RPC after error@849866855) -> (continuation of Deferred@520076599 after retry RPC after error@22627564) -> (continuation of Deferred@1914593821 after retry RPC after error@2142736625) -> (continuation of Deferred@474123317 after retry RPC after error@1467467125) -> (continuation of Deferred@1450728204 after retry RPC after error@361457962) -> (continuation of Deferred@488664202 after retry RPC after error@1947553334), errback=(continuation of Deferred@1665791915 after retry RPC after error@552074183) -> (continuation of Deferred@1696776920 after retry RPC after error@849866855) -> (continuation of Deferred@520076599 after retry RPC after error@22627564) -> (continuation of Deferred@1914593821 after retry RPC after error@2142736625) -> (continuation of Deferred@474123317 after retry RPC after error@1467467125) -> (continuation of Deferred@1450728204 after retry RPC after error@361457962) -> (continuation of Deferred@488664202 after retry RPC after error@1947553334)))
21:04:26.264 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:26.265 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:26.265 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:26.265 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:26.265 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:26.265 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:26.266 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:26.266 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:26.267 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:26.284 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:26.285 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:26.285 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:26.285 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:26.285 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:26.285 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:26.285 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:26.285 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:26.286 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:26.304 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:26.305 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:26.305 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:26.305 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:26.305 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:26.305 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:26.305 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:26.305 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:26.306 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:26.324 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:26.325 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:26.325 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:26.325 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:26.325 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:26.325 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:26.325 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:26.325 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:26.327 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:26.344 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:26.345 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:26.345 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:26.345 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:26.345 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:26.345 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:26.345 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:26.345 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:26.346 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:26.384 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:26.385 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:26.385 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:26.385 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:26.386 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:26.386 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:26.386 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:26.386 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:26.386 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:26.424 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:26.425 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:26.425 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:26.426 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:26.426 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:26.426 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:26.425 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:26.426 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:26.426 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:26.426 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:26.427 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:26.427 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:26.465 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:26.465 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:26.466 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:26.466 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:26.466 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:26.466 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:26.466 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:26.466 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:26.467 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:26.468 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=203), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [63ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [161ms] refreshing cache from master, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [162ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [162ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [163ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [163ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [200ms] refreshing cache from master, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [202ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [202ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [203ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [203ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, deferred=Deferred@1257924766(state=PENDING, result=null, callback=(continuation of Deferred@622885845 after retry RPC after error@1135804954) -> (continuation of Deferred@1213177376 after retry RPC after error@1365811503) -> (continuation of Deferred@1999668740 after retry RPC after error@742975786) -> (continuation of Deferred@407030813 after retry RPC after error@204957667) -> (continuation of Deferred@2135216327 after retry RPC after error@338589628) -> (continuation of Deferred@444752640 after retry RPC after error@767367116) -> (continuation of Deferred@1957050452 after retry RPC after error@332538603), errback=(continuation of Deferred@622885845 after retry RPC after error@1135804954) -> (continuation of Deferred@1213177376 after retry RPC after error@1365811503) -> (continuation of Deferred@1999668740 after retry RPC after error@742975786) -> (continuation of Deferred@407030813 after retry RPC after error@204957667) -> (continuation of Deferred@2135216327 after retry RPC after error@338589628) -> (continuation of Deferred@444752640 after retry RPC after error@767367116) -> (continuation of Deferred@1957050452 after retry RPC after error@332538603)))
21:04:26.468 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=203), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [63ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [161ms] refreshing cache from master, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [162ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [162ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [163ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [163ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [200ms] refreshing cache from master, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [202ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [202ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [203ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [203ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, deferred=Deferred@1257924766(state=PENDING, result=null, callback=(continuation of Deferred@622885845 after retry RPC after error@1135804954) -> (continuation of Deferred@1213177376 after retry RPC after error@1365811503) -> (continuation of Deferred@1999668740 after retry RPC after error@742975786) -> (continuation of Deferred@407030813 after retry RPC after error@204957667) -> (continuation of Deferred@2135216327 after retry RPC after error@338589628) -> (continuation of Deferred@444752640 after retry RPC after error@767367116) -> (continuation of Deferred@1957050452 after retry RPC after error@332538603), errback=(continuation of Deferred@622885845 after retry RPC after error@1135804954) -> (continuation of Deferred@1213177376 after retry RPC after error@1365811503) -> (continuation of Deferred@1999668740 after retry RPC after error@742975786) -> (continuation of Deferred@407030813 after retry RPC after error@204957667) -> (continuation of Deferred@2135216327 after retry RPC after error@338589628) -> (continuation of Deferred@444752640 after retry RPC after error@767367116) -> (continuation of Deferred@1957050452 after retry RPC after error@332538603)))
21:04:26.485 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:26.485 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:26.486 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:26.486 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:26.486 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:26.486 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:26.486 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:26.486 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:26.487 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:26.504 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:26.505 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:26.505 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:26.505 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:26.505 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:26.505 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:26.506 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:26.506 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:26.507 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:26.524 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:26.525 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:26.525 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:26.525 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:26.525 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:26.525 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:26.525 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:26.526 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:26.527 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:26.544 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:26.545 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:26.545 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:26.545 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:26.545 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:26.545 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:26.545 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:26.545 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:26.547 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:26.564 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:26.565 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:26.565 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:26.565 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:26.565 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:26.565 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:26.565 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:26.565 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:26.566 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:26.604 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:26.605 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:26.605 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:26.605 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:26.606 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:26.606 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:26.606 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:26.606 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:26.607 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:26.665 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:26.665 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:26.666 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:26.666 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:26.666 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:26.666 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:26.666 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:26.666 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:26.667 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:26.667 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=182), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [3ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [182ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [182ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [183ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [183ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, deferred=Deferred@1958211161(state=PENDING, result=null, callback=(continuation of Deferred@472503904 after retry RPC after error@800894134) -> (continuation of Deferred@859654406 after retry RPC after error@2016656414) -> (continuation of Deferred@721158062 after retry RPC after error@1625833307) -> (continuation of Deferred@969158457 after retry RPC after error@1559406419) -> (continuation of Deferred@687941100 after retry RPC after error@145933680) -> (continuation of Deferred@278372024 after retry RPC after error@1503034310), errback=(continuation of Deferred@472503904 after retry RPC after error@800894134) -> (continuation of Deferred@859654406 after retry RPC after error@2016656414) -> (continuation of Deferred@721158062 after retry RPC after error@1625833307) -> (continuation of Deferred@969158457 after retry RPC after error@1559406419) -> (continuation of Deferred@687941100 after retry RPC after error@145933680) -> (continuation of Deferred@278372024 after retry RPC after error@1503034310)))
21:04:26.668 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=182), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [3ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [182ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [182ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [183ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [183ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, deferred=Deferred@1958211161(state=PENDING, result=null, callback=(continuation of Deferred@472503904 after retry RPC after error@800894134) -> (continuation of Deferred@859654406 after retry RPC after error@2016656414) -> (continuation of Deferred@721158062 after retry RPC after error@1625833307) -> (continuation of Deferred@969158457 after retry RPC after error@1559406419) -> (continuation of Deferred@687941100 after retry RPC after error@145933680) -> (continuation of Deferred@278372024 after retry RPC after error@1503034310), errback=(continuation of Deferred@472503904 after retry RPC after error@800894134) -> (continuation of Deferred@859654406 after retry RPC after error@2016656414) -> (continuation of Deferred@721158062 after retry RPC after error@1625833307) -> (continuation of Deferred@969158457 after retry RPC after error@1559406419) -> (continuation of Deferred@687941100 after retry RPC after error@145933680) -> (continuation of Deferred@278372024 after retry RPC after error@1503034310)))
21:04:26.685 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:26.685 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:26.685 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:26.686 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:26.686 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:26.686 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:26.686 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:26.686 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:26.687 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:26.704 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:26.705 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:26.705 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:26.705 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:26.706 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:26.706 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:26.706 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:26.706 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:26.707 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:26.725 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:26.725 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:26.726 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:26.726 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:26.726 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:26.728 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:26.729 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:26.729 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:26.730 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:26.744 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:26.744 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:26.745 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:26.745 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:26.745 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:26.745 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:26.745 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:26.745 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:26.746 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:26.764 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:26.764 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:26.765 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:26.765 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:26.765 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:26.765 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:26.765 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:26.765 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:26.766 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:26.784 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:26.784 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:26.785 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:26.785 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:26.785 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:26.785 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:26.785 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:26.785 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:26.786 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:26.805 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:26.805 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:26.805 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:26.805 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:26.805 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:26.806 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:26.806 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:26.806 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:26.807 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:26.904 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:26.904 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:26.905 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:26.905 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:26.905 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:26.905 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:26.905 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:26.905 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:26.906 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:26.907 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=221), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [3ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [23ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [45ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [46ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [46ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [99ms] refreshing cache from master, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [221ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [221ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [222ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [222ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, deferred=Deferred@1032404074(state=PENDING, result=null, callback=(continuation of Deferred@20255410 after retry RPC after error@444005401) -> (continuation of Deferred@1979633422 after retry RPC after error@23119799) -> (continuation of Deferred@678032811 after retry RPC after error@89281163) -> (continuation of Deferred@106396560 after retry RPC after error@26040722) -> (continuation of Deferred@1714299045 after retry RPC after error@1912598753) -> (continuation of Deferred@1288281781 after retry RPC after error@660807543) -> (continuation of Deferred@819931190 after retry RPC after error@1051307572), errback=(continuation of Deferred@20255410 after retry RPC after error@444005401) -> (continuation of Deferred@1979633422 after retry RPC after error@23119799) -> (continuation of Deferred@678032811 after retry RPC after error@89281163) -> (continuation of Deferred@106396560 after retry RPC after error@26040722) -> (continuation of Deferred@1714299045 after retry RPC after error@1912598753) -> (continuation of Deferred@1288281781 after retry RPC after error@660807543) -> (continuation of Deferred@819931190 after retry RPC after error@1051307572)))
21:04:26.907 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=221), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [3ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [23ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [45ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [46ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [46ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [99ms] refreshing cache from master, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [221ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [221ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [222ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [222ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, deferred=Deferred@1032404074(state=PENDING, result=null, callback=(continuation of Deferred@20255410 after retry RPC after error@444005401) -> (continuation of Deferred@1979633422 after retry RPC after error@23119799) -> (continuation of Deferred@678032811 after retry RPC after error@89281163) -> (continuation of Deferred@106396560 after retry RPC after error@26040722) -> (continuation of Deferred@1714299045 after retry RPC after error@1912598753) -> (continuation of Deferred@1288281781 after retry RPC after error@660807543) -> (continuation of Deferred@819931190 after retry RPC after error@1051307572), errback=(continuation of Deferred@20255410 after retry RPC after error@444005401) -> (continuation of Deferred@1979633422 after retry RPC after error@23119799) -> (continuation of Deferred@678032811 after retry RPC after error@89281163) -> (continuation of Deferred@106396560 after retry RPC after error@26040722) -> (continuation of Deferred@1714299045 after retry RPC after error@1912598753) -> (continuation of Deferred@1288281781 after retry RPC after error@660807543) -> (continuation of Deferred@819931190 after retry RPC after error@1051307572)))
21:04:26.924 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:26.924 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:26.925 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:26.925 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:26.925 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:26.925 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:26.925 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:26.925 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:26.926 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:26.944 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:26.944 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:26.945 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:26.945 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:26.945 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:26.945 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:26.945 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:26.945 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:26.946 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:26.964 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:26.964 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:26.965 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:26.965 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:26.965 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:26.965 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:26.965 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:26.965 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:26.966 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:26.984 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:26.984 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:26.985 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:26.985 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:26.985 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:26.985 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:26.985 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:26.985 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:26.987 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:27.004 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:27.004 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:27.005 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:27.005 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:27.005 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:27.005 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:27.005 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:27.005 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:27.006 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:27.024 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:27.024 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:27.025 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:27.025 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:27.025 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:27.025 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:27.025 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:27.025 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:27.026 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:27.044 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:27.044 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:27.045 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:27.045 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:27.045 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:27.045 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:27.045 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:27.045 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:27.047 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:27.064 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:27.064 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:27.065 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:27.065 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:27.065 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:27.066 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:27.066 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:27.066 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:27.066 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:27.165 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:27.165 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:27.166 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:27.166 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:27.166 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:27.166 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:27.166 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:27.166 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:27.167 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:27.167 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=9, TimeoutTracker(timeout=250, elapsed=243), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [19ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [39ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [59ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [63ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [141ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [142ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [142ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [142ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [242ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [242ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [243ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [243ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, deferred=Deferred@2114307667(state=PENDING, result=null, callback=(continuation of Deferred@1442145840 after retry RPC after error@682170931) -> (continuation of Deferred@141170985 after retry RPC after error@1477271849) -> (continuation of Deferred@649550078 after retry RPC after error@1185920665) -> (continuation of Deferred@517233716 after retry RPC after error@761492206) -> (continuation of Deferred@823115540 after retry RPC after error@179293394) -> (continuation of Deferred@908461135 after retry RPC after error@6612071) -> (continuation of Deferred@1281297504 after retry RPC after error@954368101) -> (continuation of Deferred@1046540106 after retry RPC after error@419459901), errback=(continuation of Deferred@1442145840 after retry RPC after error@682170931) -> (continuation of Deferred@141170985 after retry RPC after error@1477271849) -> (continuation of Deferred@649550078 after retry RPC after error@1185920665) -> (continuation of Deferred@517233716 after retry RPC after error@761492206) -> (continuation of Deferred@823115540 after retry RPC after error@179293394) -> (continuation of Deferred@908461135 after retry RPC after error@6612071) -> (continuation of Deferred@1281297504 after retry RPC after error@954368101) -> (continuation of Deferred@1046540106 after retry RPC after error@419459901)))
21:04:27.168 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=9, TimeoutTracker(timeout=250, elapsed=243), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [19ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [39ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [59ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [63ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [141ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [142ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [142ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [142ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [242ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [242ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [243ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [243ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, deferred=Deferred@2114307667(state=PENDING, result=null, callback=(continuation of Deferred@1442145840 after retry RPC after error@682170931) -> (continuation of Deferred@141170985 after retry RPC after error@1477271849) -> (continuation of Deferred@649550078 after retry RPC after error@1185920665) -> (continuation of Deferred@517233716 after retry RPC after error@761492206) -> (continuation of Deferred@823115540 after retry RPC after error@179293394) -> (continuation of Deferred@908461135 after retry RPC after error@6612071) -> (continuation of Deferred@1281297504 after retry RPC after error@954368101) -> (continuation of Deferred@1046540106 after retry RPC after error@419459901), errback=(continuation of Deferred@1442145840 after retry RPC after error@682170931) -> (continuation of Deferred@141170985 after retry RPC after error@1477271849) -> (continuation of Deferred@649550078 after retry RPC after error@1185920665) -> (continuation of Deferred@517233716 after retry RPC after error@761492206) -> (continuation of Deferred@823115540 after retry RPC after error@179293394) -> (continuation of Deferred@908461135 after retry RPC after error@6612071) -> (continuation of Deferred@1281297504 after retry RPC after error@954368101) -> (continuation of Deferred@1046540106 after retry RPC after error@419459901)))
21:04:27.185 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:27.185 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:27.186 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:27.186 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:27.186 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:27.186 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:27.186 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:27.186 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:27.187 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:27.204 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:27.204 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:27.205 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:27.205 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:27.205 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:27.205 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:27.205 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:27.205 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:27.206 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:27.225 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:27.225 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:27.226 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:27.226 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:27.226 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:27.226 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:27.226 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:27.226 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:27.227 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:27.244 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:27.244 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:27.245 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:27.245 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:27.245 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:27.245 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:27.245 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:27.245 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:27.246 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:27.264 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:27.264 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:27.265 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:27.265 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:27.265 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:27.265 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:27.265 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:27.265 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:27.266 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:27.285 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:27.285 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:27.285 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:27.286 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:27.286 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:27.286 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:27.286 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:27.286 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:27.287 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:27.305 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:27.305 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:27.306 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:27.306 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:27.306 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:27.306 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:27.306 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:27.306 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:27.309 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:27.384 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:27.384 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:27.385 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:27.385 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:27.385 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:27.385 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:27.386 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:27.386 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:27.386 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:27.424 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:27.424 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:27.425 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:27.425 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:27.425 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:27.425 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:27.425 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:27.425 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:27.426 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:27.427 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=9, TimeoutTracker(timeout=250, elapsed=241), Traces: [0ms] refreshing cache from master, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [3ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [19ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [43ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [59ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [79ms] refreshing cache from master, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [125ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [125ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [201ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [202ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [202ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [241ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [241ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [242ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [242ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, deferred=Deferred@1400623583(state=PENDING, result=null, callback=(continuation of Deferred@376840410 after retry RPC after error@1155879287) -> (continuation of Deferred@521260827 after retry RPC after error@317263887) -> (continuation of Deferred@901681784 after retry RPC after error@1530794833) -> (continuation of Deferred@1254520378 after retry RPC after error@284524037) -> (continuation of Deferred@1360593713 after retry RPC after error@2056660696) -> (continuation of Deferred@60352534 after retry RPC after error@942058294) -> (continuation of Deferred@557787016 after retry RPC after error@1792304529) -> (continuation of Deferred@776530053 after retry RPC after error@591038817), errback=(continuation of Deferred@376840410 after retry RPC after error@1155879287) -> (continuation of Deferred@521260827 after retry RPC after error@317263887) -> (continuation of Deferred@901681784 after retry RPC after error@1530794833) -> (continuation of Deferred@1254520378 after retry RPC after error@284524037) -> (continuation of Deferred@1360593713 after retry RPC after error@2056660696) -> (continuation of Deferred@60352534 after retry RPC after error@942058294) -> (continuation of Deferred@557787016 after retry RPC after error@1792304529) -> (continuation of Deferred@776530053 after retry RPC after error@591038817)))
21:04:27.428 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=9, TimeoutTracker(timeout=250, elapsed=241), Traces: [0ms] refreshing cache from master, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [3ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [19ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [43ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [59ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [79ms] refreshing cache from master, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [125ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [125ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [201ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [202ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [202ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [241ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [241ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [242ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [242ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, deferred=Deferred@1400623583(state=PENDING, result=null, callback=(continuation of Deferred@376840410 after retry RPC after error@1155879287) -> (continuation of Deferred@521260827 after retry RPC after error@317263887) -> (continuation of Deferred@901681784 after retry RPC after error@1530794833) -> (continuation of Deferred@1254520378 after retry RPC after error@284524037) -> (continuation of Deferred@1360593713 after retry RPC after error@2056660696) -> (continuation of Deferred@60352534 after retry RPC after error@942058294) -> (continuation of Deferred@557787016 after retry RPC after error@1792304529) -> (continuation of Deferred@776530053 after retry RPC after error@591038817), errback=(continuation of Deferred@376840410 after retry RPC after error@1155879287) -> (continuation of Deferred@521260827 after retry RPC after error@317263887) -> (continuation of Deferred@901681784 after retry RPC after error@1530794833) -> (continuation of Deferred@1254520378 after retry RPC after error@284524037) -> (continuation of Deferred@1360593713 after retry RPC after error@2056660696) -> (continuation of Deferred@60352534 after retry RPC after error@942058294) -> (continuation of Deferred@557787016 after retry RPC after error@1792304529) -> (continuation of Deferred@776530053 after retry RPC after error@591038817)))
21:04:27.444 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:27.444 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:27.445 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:27.445 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:27.445 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:27.445 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:27.445 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:27.445 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:27.446 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:27.464 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:27.464 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:27.465 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:27.465 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:27.465 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:27.465 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:27.465 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:27.465 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:27.466 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:27.484 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:27.485 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:27.485 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:27.485 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:27.485 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:27.486 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:27.486 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:27.486 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:27.487 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:27.504 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:27.505 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:27.505 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:27.505 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:27.505 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:27.505 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:27.505 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:27.505 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:27.506 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:27.524 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:27.524 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:27.525 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:27.525 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:27.525 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:27.525 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:27.525 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:27.525 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:27.526 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:27.544 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:27.544 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:27.545 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:27.545 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:27.545 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:27.545 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:27.545 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:27.545 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:27.546 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:27.584 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:27.584 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:27.585 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:27.585 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:27.585 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:27.585 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:27.585 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:27.585 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:27.586 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:27.644 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:27.645 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:27.645 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:27.645 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:27.645 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:27.645 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:27.645 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:27.646 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:27.647 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:27.664 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:27.665 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:27.665 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:27.665 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:27.665 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:27.665 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:27.665 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:27.665 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:27.666 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:27.667 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=9, TimeoutTracker(timeout=250, elapsed=222), Traces: [0ms] refreshing cache from master, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [3ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [41ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [42ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [42ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [43ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [43ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [43ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [44ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [61ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [62ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [62ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [63ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [64ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [81ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [83ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [101ms] refreshing cache from master, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [103ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [141ms] refreshing cache from master, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [142ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [142ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [143ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [143ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [201ms] refreshing cache from master, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [202ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [202ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [202ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [202ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [204ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [204ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [221ms] refreshing cache from master, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [222ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [222ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [222ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [223ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [223ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, deferred=Deferred@1686076872(state=PENDING, result=null, callback=(continuation of Deferred@2062204074 after retry RPC after error@1607144676) -> (continuation of Deferred@2029433814 after retry RPC after error@1744306237) -> (continuation of Deferred@1213788036 after retry RPC after error@1328787039) -> (continuation of Deferred@35052652 after retry RPC after error@625532411) -> (continuation of Deferred@711143961 after retry RPC after error@2020614236) -> (continuation of Deferred@1286694530 after retry RPC after error@796249578) -> (continuation of Deferred@481673703 after retry RPC after error@1698590957) -> (continuation of Deferred@578592702 after retry RPC after error@602348678), errback=(continuation of Deferred@2062204074 after retry RPC after error@1607144676) -> (continuation of Deferred@2029433814 after retry RPC after error@1744306237) -> (continuation of Deferred@1213788036 after retry RPC after error@1328787039) -> (continuation of Deferred@35052652 after retry RPC after error@625532411) -> (continuation of Deferred@711143961 after retry RPC after error@2020614236) -> (continuation of Deferred@1286694530 after retry RPC after error@796249578) -> (continuation of Deferred@481673703 after retry RPC after error@1698590957) -> (continuation of Deferred@578592702 after retry RPC after error@602348678)))
21:04:27.667 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=9, TimeoutTracker(timeout=250, elapsed=222), Traces: [0ms] refreshing cache from master, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [3ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [41ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [42ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [42ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [43ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [43ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [43ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [44ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [61ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [62ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [62ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [63ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [64ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [81ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [83ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [101ms] refreshing cache from master, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [103ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [141ms] refreshing cache from master, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [142ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [142ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [143ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [143ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [201ms] refreshing cache from master, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [202ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [202ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [202ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [202ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [204ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [204ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [221ms] refreshing cache from master, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [222ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [222ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [222ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [223ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [223ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, deferred=Deferred@1686076872(state=PENDING, result=null, callback=(continuation of Deferred@2062204074 after retry RPC after error@1607144676) -> (continuation of Deferred@2029433814 after retry RPC after error@1744306237) -> (continuation of Deferred@1213788036 after retry RPC after error@1328787039) -> (continuation of Deferred@35052652 after retry RPC after error@625532411) -> (continuation of Deferred@711143961 after retry RPC after error@2020614236) -> (continuation of Deferred@1286694530 after retry RPC after error@796249578) -> (continuation of Deferred@481673703 after retry RPC after error@1698590957) -> (continuation of Deferred@578592702 after retry RPC after error@602348678), errback=(continuation of Deferred@2062204074 after retry RPC after error@1607144676) -> (continuation of Deferred@2029433814 after retry RPC after error@1744306237) -> (continuation of Deferred@1213788036 after retry RPC after error@1328787039) -> (continuation of Deferred@35052652 after retry RPC after error@625532411) -> (continuation of Deferred@711143961 after retry RPC after error@2020614236) -> (continuation of Deferred@1286694530 after retry RPC after error@796249578) -> (continuation of Deferred@481673703 after retry RPC after error@1698590957) -> (continuation of Deferred@578592702 after retry RPC after error@602348678)))
21:04:27.684 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:27.685 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:27.685 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:27.685 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:27.685 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:27.685 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:27.685 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:27.685 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:27.686 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:27.704 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:27.705 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:27.705 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:27.705 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:27.705 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:27.705 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:27.705 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:27.705 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:27.706 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:27.724 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:27.725 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:27.725 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:27.725 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:27.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:27.725759 23569 master_proxy_rpc.cc:177] Failed to determine new leader Master: Timed out: ConnectToClusterRpc(addrs: 127.22.168.62:37239,127.22.168.61:32947,127.22.168.60:33247, num_attempts: 97) passed its deadline: Not found: no leader found: ConnectToClusterRpc(addrs: 127.22.168.62:37239,127.22.168.61:32947,127.22.168.60:33247, num_attempts: 1)
21:04:27.725 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:27.726 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:27.726 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:27.726 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:27.726 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:27.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:27.730182 23569 meta_cache.cc:788] Timed out: LookupRpcById { tablet: '01aa0e192c434b468ec7ed90d48563b3', attempt: 2 } failed: LookupRpcById timed out after deadline expired: LookupRpcById { tablet: '01aa0e192c434b468ec7ed90d48563b3', attempt: 2 } passed its deadline: Timed out: ConnectToClusterRpc(addrs: 127.22.168.62:37239,127.22.168.61:32947,127.22.168.60:33247, num_attempts: 97) passed its deadline: Not found: no leader found: ConnectToClusterRpc(addrs: 127.22.168.62:37239,127.22.168.61:32947,127.22.168.60:33247, num_attempts: 1)
21:04:27.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:27.732612 23569 master_proxy_rpc.cc:203] Re-attempting LookupRpcById request to leader Master (127.22.168.60:33247)
21:04:27.744 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:27.745 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:27.745 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:27.745 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:27.745 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:27.746 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:27.746 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:27.746 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:27.746 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:27.764 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:27.764 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:27.765 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:27.765 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:27.765 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:27.765 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:27.765 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:27.765 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:27.766 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:27.804 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:27.805 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:27.805 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:27.805 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:27.806 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:27.806 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:27.806 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:27.806 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:27.807 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:27.864 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:27.864 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:27.865 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:27.865 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:27.865 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:27.865 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:27.865 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:27.865 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:27.866 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:27.866 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=181), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [123ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [179ms] refreshing cache from master, [179ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [181ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [181ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [182ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, deferred=Deferred@81083662(state=PENDING, result=null, callback=(continuation of Deferred@1769143904 after retry RPC after error@1098741817) -> (continuation of Deferred@810102833 after retry RPC after error@500778619) -> (continuation of Deferred@1671834206 after retry RPC after error@1236282019) -> (continuation of Deferred@805389084 after retry RPC after error@1740382713) -> (continuation of Deferred@890901386 after retry RPC after error@2109199032) -> (continuation of Deferred@1192523697 after retry RPC after error@1101079875), errback=(continuation of Deferred@1769143904 after retry RPC after error@1098741817) -> (continuation of Deferred@810102833 after retry RPC after error@500778619) -> (continuation of Deferred@1671834206 after retry RPC after error@1236282019) -> (continuation of Deferred@805389084 after retry RPC after error@1740382713) -> (continuation of Deferred@890901386 after retry RPC after error@2109199032) -> (continuation of Deferred@1192523697 after retry RPC after error@1101079875)))
21:04:27.867 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=181), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [123ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [179ms] refreshing cache from master, [179ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [181ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [181ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [182ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, deferred=Deferred@81083662(state=PENDING, result=null, callback=(continuation of Deferred@1769143904 after retry RPC after error@1098741817) -> (continuation of Deferred@810102833 after retry RPC after error@500778619) -> (continuation of Deferred@1671834206 after retry RPC after error@1236282019) -> (continuation of Deferred@805389084 after retry RPC after error@1740382713) -> (continuation of Deferred@890901386 after retry RPC after error@2109199032) -> (continuation of Deferred@1192523697 after retry RPC after error@1101079875), errback=(continuation of Deferred@1769143904 after retry RPC after error@1098741817) -> (continuation of Deferred@810102833 after retry RPC after error@500778619) -> (continuation of Deferred@1671834206 after retry RPC after error@1236282019) -> (continuation of Deferred@805389084 after retry RPC after error@1740382713) -> (continuation of Deferred@890901386 after retry RPC after error@2109199032) -> (continuation of Deferred@1192523697 after retry RPC after error@1101079875)))
21:04:27.885 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:27.885 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:27.885 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:27.885 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:27.885 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:27.886 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:27.886 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:27.886 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:27.887 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:27.904 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:27.905 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:27.905 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:27.905 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:27.905 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:27.905 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:27.905 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:27.905 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:27.906 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:27.924 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:27.924 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:27.925 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:27.925 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:27.925 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:27.925 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:27.925 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:27.925 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:27.926 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:27.944 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:27.944 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:27.945 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:27.945 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:27.945 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:27.945 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:27.945 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:27.945 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:27.946 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:27.964 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:27.964 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:27.965 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:27.965 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:27.965 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:27.965 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:27.965 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:27.965 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:27.966 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:27.984 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:27.984 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:27.985 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:27.985 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:27.985 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:27.985 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:27.985 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:27.985 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:27.986 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:28.004 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:28.004 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:28.005 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:28.005 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:28.005 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:28.005 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:28.005 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:28.005 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:28.006 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:28.085 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:28.085 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:28.086 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:28.086 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:28.086 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:28.086 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:28.086 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:28.086 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:28.087 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:28.104 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:28.104 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:28.105 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:28.105 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:28.105 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:28.105 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:28.105 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:28.106 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:28.107 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:28.107 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=9, TimeoutTracker(timeout=250, elapsed=222), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [200ms] refreshing cache from master, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [202ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [202ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [203ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [203ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [221ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [221ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [223ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [223ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, deferred=Deferred@1863557660(state=PENDING, result=null, callback=(continuation of Deferred@57596634 after retry RPC after error@1016244215) -> (continuation of Deferred@1139020363 after retry RPC after error@823483241) -> (continuation of Deferred@1155071899 after retry RPC after error@1843156456) -> (continuation of Deferred@248079442 after retry RPC after error@354203889) -> (continuation of Deferred@373012974 after retry RPC after error@432528852) -> (continuation of Deferred@1550438730 after retry RPC after error@742531334) -> (continuation of Deferred@1736251817 after retry RPC after error@1089688047) -> (continuation of Deferred@1384266554 after retry RPC after error@1878358653), errback=(continuation of Deferred@57596634 after retry RPC after error@1016244215) -> (continuation of Deferred@1139020363 after retry RPC after error@823483241) -> (continuation of Deferred@1155071899 after retry RPC after error@1843156456) -> (continuation of Deferred@248079442 after retry RPC after error@354203889) -> (continuation of Deferred@373012974 after retry RPC after error@432528852) -> (continuation of Deferred@1550438730 after retry RPC after error@742531334) -> (continuation of Deferred@1736251817 after retry RPC after error@1089688047) -> (continuation of Deferred@1384266554 after retry RPC after error@1878358653)))
21:04:28.108 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=9, TimeoutTracker(timeout=250, elapsed=222), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [200ms] refreshing cache from master, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [202ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [202ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [203ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [203ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [221ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [221ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [223ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [223ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, deferred=Deferred@1863557660(state=PENDING, result=null, callback=(continuation of Deferred@57596634 after retry RPC after error@1016244215) -> (continuation of Deferred@1139020363 after retry RPC after error@823483241) -> (continuation of Deferred@1155071899 after retry RPC after error@1843156456) -> (continuation of Deferred@248079442 after retry RPC after error@354203889) -> (continuation of Deferred@373012974 after retry RPC after error@432528852) -> (continuation of Deferred@1550438730 after retry RPC after error@742531334) -> (continuation of Deferred@1736251817 after retry RPC after error@1089688047) -> (continuation of Deferred@1384266554 after retry RPC after error@1878358653), errback=(continuation of Deferred@57596634 after retry RPC after error@1016244215) -> (continuation of Deferred@1139020363 after retry RPC after error@823483241) -> (continuation of Deferred@1155071899 after retry RPC after error@1843156456) -> (continuation of Deferred@248079442 after retry RPC after error@354203889) -> (continuation of Deferred@373012974 after retry RPC after error@432528852) -> (continuation of Deferred@1550438730 after retry RPC after error@742531334) -> (continuation of Deferred@1736251817 after retry RPC after error@1089688047) -> (continuation of Deferred@1384266554 after retry RPC after error@1878358653)))
21:04:28.124 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:28.125 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:28.125 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:28.125 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:28.125 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:28.126 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:28.126 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:28.126 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:28.127 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:28.144 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:28.145 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:28.145 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:28.145 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:28.146 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:28.146 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:28.146 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:28.146 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:28.146 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:28.164 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:28.165 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:28.165 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:28.165 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:28.165 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:28.165 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:28.166 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:28.166 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:28.166 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:28.184 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:28.185 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:28.185 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:28.185 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:28.185 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:28.185 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:28.185 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:28.186 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:28.186 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:28.204 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:28.205 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:28.205 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:28.205 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:28.205 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:28.205 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:28.206 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:28.206 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:28.206 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:28.245 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:28.245 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:28.246 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:28.246 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:28.246 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:28.246 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:28.246 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:28.246 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:28.247 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:28.265 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:28.265 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:28.266 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:28.266 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:28.266 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:28.266 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:28.266 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:28.266 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:28.266 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:28.285 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:28.285 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:28.286 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:28.286 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:28.286 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:28.286 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:28.286 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:28.286 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:28.287 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:28.304 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:28.305 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:28.305 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:28.305 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:28.305 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:28.305 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:28.306 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:28.306 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:28.306 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:28.307 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=9, TimeoutTracker(timeout=250, elapsed=182), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [142ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [142ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [142ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [162ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [162ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [162ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [163ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [181ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [181ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [182ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, deferred=Deferred@785308885(state=PENDING, result=null, callback=(continuation of Deferred@1206128947 after retry RPC after error@1926297710) -> (continuation of Deferred@1843928717 after retry RPC after error@899174289) -> (continuation of Deferred@1385344408 after retry RPC after error@670317612) -> (continuation of Deferred@180703785 after retry RPC after error@2111205415) -> (continuation of Deferred@988806248 after retry RPC after error@1014805061) -> (continuation of Deferred@470346745 after retry RPC after error@1376904983) -> (continuation of Deferred@391662977 after retry RPC after error@1324297141) -> (continuation of Deferred@358827698 after retry RPC after error@1209479766), errback=(continuation of Deferred@1206128947 after retry RPC after error@1926297710) -> (continuation of Deferred@1843928717 after retry RPC after error@899174289) -> (continuation of Deferred@1385344408 after retry RPC after error@670317612) -> (continuation of Deferred@180703785 after retry RPC after error@2111205415) -> (continuation of Deferred@988806248 after retry RPC after error@1014805061) -> (continuation of Deferred@470346745 after retry RPC after error@1376904983) -> (continuation of Deferred@391662977 after retry RPC after error@1324297141) -> (continuation of Deferred@358827698 after retry RPC after error@1209479766)))
21:04:28.307 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=9, TimeoutTracker(timeout=250, elapsed=182), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [142ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [142ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [142ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [162ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [162ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [162ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [163ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [181ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [181ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [182ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, deferred=Deferred@785308885(state=PENDING, result=null, callback=(continuation of Deferred@1206128947 after retry RPC after error@1926297710) -> (continuation of Deferred@1843928717 after retry RPC after error@899174289) -> (continuation of Deferred@1385344408 after retry RPC after error@670317612) -> (continuation of Deferred@180703785 after retry RPC after error@2111205415) -> (continuation of Deferred@988806248 after retry RPC after error@1014805061) -> (continuation of Deferred@470346745 after retry RPC after error@1376904983) -> (continuation of Deferred@391662977 after retry RPC after error@1324297141) -> (continuation of Deferred@358827698 after retry RPC after error@1209479766), errback=(continuation of Deferred@1206128947 after retry RPC after error@1926297710) -> (continuation of Deferred@1843928717 after retry RPC after error@899174289) -> (continuation of Deferred@1385344408 after retry RPC after error@670317612) -> (continuation of Deferred@180703785 after retry RPC after error@2111205415) -> (continuation of Deferred@988806248 after retry RPC after error@1014805061) -> (continuation of Deferred@470346745 after retry RPC after error@1376904983) -> (continuation of Deferred@391662977 after retry RPC after error@1324297141) -> (continuation of Deferred@358827698 after retry RPC after error@1209479766)))
21:04:28.324 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:28.325 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:28.325 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:28.325 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:28.325 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:28.325 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:28.325 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:28.325 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:28.326 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:28.344 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:28.344 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:28.345 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:28.345 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:28.345 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:28.345 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:28.345 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:28.345 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:28.346 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:28.364 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:28.364 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:28.365 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:28.365 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:28.365 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:28.365 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:28.365 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:28.365 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:28.366 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:28.384 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:28.384 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:28.385 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:28.385 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:28.385 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:28.385 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:28.385 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:28.385 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:28.386 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:28.404 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:28.404 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:28.405 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:28.405 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:28.405 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:28.405 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:28.405 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:28.405 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:28.406 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:28.444 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:28.444 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:28.445 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:28.445 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:28.445 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:28.445 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:28.445 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:28.446 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:28.447 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:28.484 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:28.484 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:28.485 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:28.485 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:28.485 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:28.485 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:28.485 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:28.486 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:28.486 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:28.487 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=162), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [79ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [119ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [161ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [162ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [162ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, deferred=Deferred@176677511(state=PENDING, result=null, callback=(continuation of Deferred@140891271 after retry RPC after error@1700606726) -> (continuation of Deferred@1345145700 after retry RPC after error@1695225916) -> (continuation of Deferred@1139554806 after retry RPC after error@1327888270) -> (continuation of Deferred@1978262373 after retry RPC after error@1209978652) -> (continuation of Deferred@1780448352 after retry RPC after error@182877994) -> (continuation of Deferred@816732120 after retry RPC after error@251009278), errback=(continuation of Deferred@140891271 after retry RPC after error@1700606726) -> (continuation of Deferred@1345145700 after retry RPC after error@1695225916) -> (continuation of Deferred@1139554806 after retry RPC after error@1327888270) -> (continuation of Deferred@1978262373 after retry RPC after error@1209978652) -> (continuation of Deferred@1780448352 after retry RPC after error@182877994) -> (continuation of Deferred@816732120 after retry RPC after error@251009278)))
21:04:28.487 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=162), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [79ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [119ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [161ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [162ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [162ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, deferred=Deferred@176677511(state=PENDING, result=null, callback=(continuation of Deferred@140891271 after retry RPC after error@1700606726) -> (continuation of Deferred@1345145700 after retry RPC after error@1695225916) -> (continuation of Deferred@1139554806 after retry RPC after error@1327888270) -> (continuation of Deferred@1978262373 after retry RPC after error@1209978652) -> (continuation of Deferred@1780448352 after retry RPC after error@182877994) -> (continuation of Deferred@816732120 after retry RPC after error@251009278), errback=(continuation of Deferred@140891271 after retry RPC after error@1700606726) -> (continuation of Deferred@1345145700 after retry RPC after error@1695225916) -> (continuation of Deferred@1139554806 after retry RPC after error@1327888270) -> (continuation of Deferred@1978262373 after retry RPC after error@1209978652) -> (continuation of Deferred@1780448352 after retry RPC after error@182877994) -> (continuation of Deferred@816732120 after retry RPC after error@251009278)))
21:04:28.504 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:28.504 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:28.505 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:28.505 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:28.505 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:28.505 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:28.505 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:28.505 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:28.506 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:28.524 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:28.524 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:28.525 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:28.525 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:28.526 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:28.526 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:28.526 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:28.526 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:28.526 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:28.544 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:28.544 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:28.545 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:28.545 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:28.545 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:28.545 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:28.545 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:28.545 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:28.546 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:28.564 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:28.564 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:28.565 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:28.565 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:28.565 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:28.565 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:28.565 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:28.565 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:28.566 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:28.585 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:28.585 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:28.585 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:28.585 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:28.586 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:28.586 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:28.586 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:28.586 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:28.587 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:28.604 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:28.605 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:28.605 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:28.606 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:28.606 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:28.607 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:28.607 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:28.607 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:28.607 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:28.624 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:28.625 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:28.625 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:28.625 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:28.625 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:28.625 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:28.625 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:28.625 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:28.626 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:28.684 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:28.685 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:28.685 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:28.685 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:28.685 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:28.685 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:28.686 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:28.686 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:28.686 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:28.687 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=182), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [59ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [103ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [103ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [103ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [181ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [182ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [182ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, deferred=Deferred@1061735051(state=PENDING, result=null, callback=(continuation of Deferred@454781871 after retry RPC after error@676965206) -> (continuation of Deferred@1978471845 after retry RPC after error@210273537) -> (continuation of Deferred@1284423980 after retry RPC after error@703119600) -> (continuation of Deferred@875931733 after retry RPC after error@2086503263) -> (continuation of Deferred@1180744281 after retry RPC after error@694164773) -> (continuation of Deferred@814645084 after retry RPC after error@551109993) -> (continuation of Deferred@1663598937 after retry RPC after error@1242050860), errback=(continuation of Deferred@454781871 after retry RPC after error@676965206) -> (continuation of Deferred@1978471845 after retry RPC after error@210273537) -> (continuation of Deferred@1284423980 after retry RPC after error@703119600) -> (continuation of Deferred@875931733 after retry RPC after error@2086503263) -> (continuation of Deferred@1180744281 after retry RPC after error@694164773) -> (continuation of Deferred@814645084 after retry RPC after error@551109993) -> (continuation of Deferred@1663598937 after retry RPC after error@1242050860)))
21:04:28.687 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=182), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [59ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [103ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [103ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [103ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [181ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [182ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [182ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, deferred=Deferred@1061735051(state=PENDING, result=null, callback=(continuation of Deferred@454781871 after retry RPC after error@676965206) -> (continuation of Deferred@1978471845 after retry RPC after error@210273537) -> (continuation of Deferred@1284423980 after retry RPC after error@703119600) -> (continuation of Deferred@875931733 after retry RPC after error@2086503263) -> (continuation of Deferred@1180744281 after retry RPC after error@694164773) -> (continuation of Deferred@814645084 after retry RPC after error@551109993) -> (continuation of Deferred@1663598937 after retry RPC after error@1242050860), errback=(continuation of Deferred@454781871 after retry RPC after error@676965206) -> (continuation of Deferred@1978471845 after retry RPC after error@210273537) -> (continuation of Deferred@1284423980 after retry RPC after error@703119600) -> (continuation of Deferred@875931733 after retry RPC after error@2086503263) -> (continuation of Deferred@1180744281 after retry RPC after error@694164773) -> (continuation of Deferred@814645084 after retry RPC after error@551109993) -> (continuation of Deferred@1663598937 after retry RPC after error@1242050860)))
21:04:28.704 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:28.705 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:28.705 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:28.705 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:28.705 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:28.705 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:28.705 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:28.706 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:28.706 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:28.724 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:28.724 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:28.725 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:28.725 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:28.725 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:28.725 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:28.725 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:28.725 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:28.726 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:28.744 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:28.744 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:28.745 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:28.745 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:28.745 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:28.745 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:28.745 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:28.745 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:28.746 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:28.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:28.760944 24311 raft_consensus.cc:493] T 00000000000000000000000000000000 P eb664add87f44013b596f4cc01c9a41c [term 4 FOLLOWER]: Starting pre-election (detected failure of leader c520077acf3341e29447c6c2b8d3f74c)
21:04:28.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:28.761215 24311 raft_consensus.cc:515] T 00000000000000000000000000000000 P eb664add87f44013b596f4cc01c9a41c [term 4 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e9dd218ca2244f5a934aac2b7e58eb1d" member_type: VOTER last_known_addr { host: "127.22.168.62" port: 37239 } } peers { permanent_uuid: "c520077acf3341e29447c6c2b8d3f74c" member_type: VOTER last_known_addr { host: "127.22.168.61" port: 32947 } } peers { permanent_uuid: "eb664add87f44013b596f4cc01c9a41c" member_type: VOTER last_known_addr { host: "127.22.168.60" port: 33247 } }
21:04:28.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:28.762044 24311 leader_election.cc:290] T 00000000000000000000000000000000 P eb664add87f44013b596f4cc01c9a41c [CANDIDATE]: Term 5 pre-election: Requested pre-vote from peers e9dd218ca2244f5a934aac2b7e58eb1d (127.22.168.62:37239), c520077acf3341e29447c6c2b8d3f74c (127.22.168.61:32947)
21:04:28.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:28.763092 24086 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.22.168.62:37239: connect: Connection refused (error 111) [suppressed 5 similar messages]
21:04:28.764 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:28.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:28.764374 24089 leader_election.cc:336] T 00000000000000000000000000000000 P eb664add87f44013b596f4cc01c9a41c [CANDIDATE]: Term 5 pre-election: RPC error from VoteRequest() call to peer c520077acf3341e29447c6c2b8d3f74c (127.22.168.61:32947): Network error: Client connection negotiation failed: client connection to 127.22.168.61:32947: connect: Connection refused (error 111)
21:04:28.764 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:28.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:28.764568 24086 leader_election.cc:336] T 00000000000000000000000000000000 P eb664add87f44013b596f4cc01c9a41c [CANDIDATE]: Term 5 pre-election: RPC error from VoteRequest() call to peer e9dd218ca2244f5a934aac2b7e58eb1d (127.22.168.62:37239): Network error: Client connection negotiation failed: client connection to 127.22.168.62:37239: connect: Connection refused (error 111)
21:04:28.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:28.764855 24086 leader_election.cc:304] T 00000000000000000000000000000000 P eb664add87f44013b596f4cc01c9a41c [CANDIDATE]: Term 5 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: eb664add87f44013b596f4cc01c9a41c; no voters: c520077acf3341e29447c6c2b8d3f74c, e9dd218ca2244f5a934aac2b7e58eb1d
21:04:28.765 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:28.765 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:28.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:28.765197 24311 raft_consensus.cc:2749] T 00000000000000000000000000000000 P eb664add87f44013b596f4cc01c9a41c [term 4 FOLLOWER]: Leader pre-election lost for term 5. Reason: could not achieve majority
21:04:28.765 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:28.765 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:28.765 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:28.765 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:28.766 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:28.784 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:28.784 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:28.785 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:28.785 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:28.785 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:28.785 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:28.785 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:28.785 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:28.786 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:28.804 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:28.804 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:28.805 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:28.805 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:28.805 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:28.805 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:28.805 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:28.805 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:28.806 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:28.864 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:28.864 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:28.865 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:28.865 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:28.865 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:28.865 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:28.865 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:28.865 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:28.866 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:28.884 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:28.884 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:28.885 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:28.885 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:28.885 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:28.885 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:28.885 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:28.885 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:28.886 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:28.886 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=181), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [79ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [99ms] refreshing cache from master, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [161ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [161ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [162ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [181ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [181ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [182ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, deferred=Deferred@129858211(state=PENDING, result=null, callback=(continuation of Deferred@1184732028 after retry RPC after error@362681326) -> (continuation of Deferred@1547938304 after retry RPC after error@818391881) -> (continuation of Deferred@451659870 after retry RPC after error@2131736974) -> (continuation of Deferred@1046286277 after retry RPC after error@860714660) -> (continuation of Deferred@1998647723 after retry RPC after error@1977742600) -> (continuation of Deferred@588578646 after retry RPC after error@2067023302) -> (continuation of Deferred@199839394 after retry RPC after error@1910158178), errback=(continuation of Deferred@1184732028 after retry RPC after error@362681326) -> (continuation of Deferred@1547938304 after retry RPC after error@818391881) -> (continuation of Deferred@451659870 after retry RPC after error@2131736974) -> (continuation of Deferred@1046286277 after retry RPC after error@860714660) -> (continuation of Deferred@1998647723 after retry RPC after error@1977742600) -> (continuation of Deferred@588578646 after retry RPC after error@2067023302) -> (continuation of Deferred@199839394 after retry RPC after error@1910158178)))
21:04:28.887 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=181), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [79ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [99ms] refreshing cache from master, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [161ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [161ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [162ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [181ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [181ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [182ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, deferred=Deferred@129858211(state=PENDING, result=null, callback=(continuation of Deferred@1184732028 after retry RPC after error@362681326) -> (continuation of Deferred@1547938304 after retry RPC after error@818391881) -> (continuation of Deferred@451659870 after retry RPC after error@2131736974) -> (continuation of Deferred@1046286277 after retry RPC after error@860714660) -> (continuation of Deferred@1998647723 after retry RPC after error@1977742600) -> (continuation of Deferred@588578646 after retry RPC after error@2067023302) -> (continuation of Deferred@199839394 after retry RPC after error@1910158178), errback=(continuation of Deferred@1184732028 after retry RPC after error@362681326) -> (continuation of Deferred@1547938304 after retry RPC after error@818391881) -> (continuation of Deferred@451659870 after retry RPC after error@2131736974) -> (continuation of Deferred@1046286277 after retry RPC after error@860714660) -> (continuation of Deferred@1998647723 after retry RPC after error@1977742600) -> (continuation of Deferred@588578646 after retry RPC after error@2067023302) -> (continuation of Deferred@199839394 after retry RPC after error@1910158178)))
21:04:28.904 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:28.904 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:28.905 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:28.905 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:28.905 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:28.905 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:28.905 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:28.905 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:28.906 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:28.924 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:28.924 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:28.925 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:28.925 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:28.925 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:28.925 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:28.925 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:28.925 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:28.926 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:28.944 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:28.944 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:28.945 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:28.945 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:28.945 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:28.945 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:28.945 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:28.945 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:28.946 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:28.964 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:28.964 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:28.965 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:28.965 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:28.965 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:28.965 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:28.965 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:28.965 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:28.966 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:28.984 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:28.984 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:28.985 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:28.985 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:28.985 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:28.985 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:28.985 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:28.985 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:28.986 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:29.004 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:29.004 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:29.005 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:29.005 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:29.005 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:29.005 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:29.005 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:29.005 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:29.006 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:29.044 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:29.044 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:29.045 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:29.045 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:29.045 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:29.045 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:29.045 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:29.045 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:29.046 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:29.064 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:29.064 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:29.065 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:29.065 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:29.065 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:29.065 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:29.065 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:29.065 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:29.066 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:29.066 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=162), Traces: [0ms] refreshing cache from master, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [3ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [20ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [23ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [41ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [43ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [61ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [63ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [80ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [83ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [100ms] refreshing cache from master, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [103ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [140ms] refreshing cache from master, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [142ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [142ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [143ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [143ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [160ms] refreshing cache from master, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [162ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [162ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [163ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [163ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, deferred=Deferred@420012709(state=PENDING, result=null, callback=(continuation of Deferred@2135609260 after retry RPC after error@924708332) -> (continuation of Deferred@349158193 after retry RPC after error@1242332871) -> (continuation of Deferred@1670573957 after retry RPC after error@1001917590) -> (continuation of Deferred@1380400867 after retry RPC after error@2073331754) -> (continuation of Deferred@252620314 after retry RPC after error@1957285841) -> (continuation of Deferred@527587201 after retry RPC after error@1081629085) -> (continuation of Deferred@981221878 after retry RPC after error@9553455), errback=(continuation of Deferred@2135609260 after retry RPC after error@924708332) -> (continuation of Deferred@349158193 after retry RPC after error@1242332871) -> (continuation of Deferred@1670573957 after retry RPC after error@1001917590) -> (continuation of Deferred@1380400867 after retry RPC after error@2073331754) -> (continuation of Deferred@252620314 after retry RPC after error@1957285841) -> (continuation of Deferred@527587201 after retry RPC after error@1081629085) -> (continuation of Deferred@981221878 after retry RPC after error@9553455)))
21:04:29.067 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=162), Traces: [0ms] refreshing cache from master, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [3ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [20ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [23ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [41ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [43ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [61ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [63ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [80ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [83ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [100ms] refreshing cache from master, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [103ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [140ms] refreshing cache from master, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [142ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [142ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [143ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [143ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [160ms] refreshing cache from master, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [162ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [162ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [163ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [163ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, deferred=Deferred@420012709(state=PENDING, result=null, callback=(continuation of Deferred@2135609260 after retry RPC after error@924708332) -> (continuation of Deferred@349158193 after retry RPC after error@1242332871) -> (continuation of Deferred@1670573957 after retry RPC after error@1001917590) -> (continuation of Deferred@1380400867 after retry RPC after error@2073331754) -> (continuation of Deferred@252620314 after retry RPC after error@1957285841) -> (continuation of Deferred@527587201 after retry RPC after error@1081629085) -> (continuation of Deferred@981221878 after retry RPC after error@9553455), errback=(continuation of Deferred@2135609260 after retry RPC after error@924708332) -> (continuation of Deferred@349158193 after retry RPC after error@1242332871) -> (continuation of Deferred@1670573957 after retry RPC after error@1001917590) -> (continuation of Deferred@1380400867 after retry RPC after error@2073331754) -> (continuation of Deferred@252620314 after retry RPC after error@1957285841) -> (continuation of Deferred@527587201 after retry RPC after error@1081629085) -> (continuation of Deferred@981221878 after retry RPC after error@9553455)))
21:04:29.084 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:29.084 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:29.085 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:29.085 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:29.085 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:29.085 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:29.085 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:29.085 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:29.086 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:29.104 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:29.104 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:29.105 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:29.105 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:29.105 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:29.105 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:29.105 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:29.105 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:29.106 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:29.124 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:29.124 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:29.125 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:29.125 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:29.125 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:29.125 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:29.125 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:29.125 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:29.126 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:29.144 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:29.144 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:29.145 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:29.145 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:29.145 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:29.145 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:29.145 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:29.145 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:29.146 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:29.164 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:29.164 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:29.164 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:29.165 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:29.165 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:29.165 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:29.165 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:29.165 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:29.166 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:29.185 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:29.185 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:29.185 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:29.185 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:29.185 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:29.185 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:29.186 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:29.186 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:29.186 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:29.245 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:29.245 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:29.245 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:29.245 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:29.245 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:29.245 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:29.246 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:29.246 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:29.247 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:29.247 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=163), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [19ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [59ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [79ms] refreshing cache from master, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [161ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [162ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [162ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [163ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, deferred=Deferred@1265328138(state=PENDING, result=null, callback=(continuation of Deferred@745529893 after retry RPC after error@686325465) -> (continuation of Deferred@885853191 after retry RPC after error@615924132) -> (continuation of Deferred@1998315997 after retry RPC after error@485377873) -> (continuation of Deferred@1084214783 after retry RPC after error@1446125353) -> (continuation of Deferred@2123303227 after retry RPC after error@280300916) -> (continuation of Deferred@788253352 after retry RPC after error@1760969481), errback=(continuation of Deferred@745529893 after retry RPC after error@686325465) -> (continuation of Deferred@885853191 after retry RPC after error@615924132) -> (continuation of Deferred@1998315997 after retry RPC after error@485377873) -> (continuation of Deferred@1084214783 after retry RPC after error@1446125353) -> (continuation of Deferred@2123303227 after retry RPC after error@280300916) -> (continuation of Deferred@788253352 after retry RPC after error@1760969481)))
21:04:29.247 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=163), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [19ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [59ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [79ms] refreshing cache from master, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [161ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [162ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [162ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [163ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, deferred=Deferred@1265328138(state=PENDING, result=null, callback=(continuation of Deferred@745529893 after retry RPC after error@686325465) -> (continuation of Deferred@885853191 after retry RPC after error@615924132) -> (continuation of Deferred@1998315997 after retry RPC after error@485377873) -> (continuation of Deferred@1084214783 after retry RPC after error@1446125353) -> (continuation of Deferred@2123303227 after retry RPC after error@280300916) -> (continuation of Deferred@788253352 after retry RPC after error@1760969481), errback=(continuation of Deferred@745529893 after retry RPC after error@686325465) -> (continuation of Deferred@885853191 after retry RPC after error@615924132) -> (continuation of Deferred@1998315997 after retry RPC after error@485377873) -> (continuation of Deferred@1084214783 after retry RPC after error@1446125353) -> (continuation of Deferred@2123303227 after retry RPC after error@280300916) -> (continuation of Deferred@788253352 after retry RPC after error@1760969481)))
21:04:29.264 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:29.265 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:29.265 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:29.265 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:29.265 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:29.266 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:29.266 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:29.266 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:29.267 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:29.284 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:29.285 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:29.285 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:29.285 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:29.285 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:29.285 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:29.285 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:29.285 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:29.286 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:29.304 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:29.304 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:29.305 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:29.305 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:29.305 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:29.305 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:29.305 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:29.305 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:29.306 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:29.324 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:29.324 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:29.325 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:29.325 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:29.325 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:29.325 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:29.325 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:29.325 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:29.326 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:29.344 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:29.344 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:29.345 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:29.345 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:29.345 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:29.345 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:29.345 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:29.345 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:29.346 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:29.384 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:29.385 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:29.385 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:29.385 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:29.385 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:29.385 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:29.385 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:29.386 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:29.386 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:29.445 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:29.445 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:29.446 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:29.446 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:29.446 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:29.446 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:29.446 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:29.446 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:29.447 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:29.447 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=182), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [3ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [182ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [182ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [183ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [183ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, deferred=Deferred@1727414181(state=PENDING, result=null, callback=(continuation of Deferred@354577675 after retry RPC after error@1155847070) -> (continuation of Deferred@2063561184 after retry RPC after error@757293581) -> (continuation of Deferred@737085591 after retry RPC after error@1232426583) -> (continuation of Deferred@1216630504 after retry RPC after error@2114027263) -> (continuation of Deferred@771324440 after retry RPC after error@1314735646) -> (continuation of Deferred@616866355 after retry RPC after error@1372047058), errback=(continuation of Deferred@354577675 after retry RPC after error@1155847070) -> (continuation of Deferred@2063561184 after retry RPC after error@757293581) -> (continuation of Deferred@737085591 after retry RPC after error@1232426583) -> (continuation of Deferred@1216630504 after retry RPC after error@2114027263) -> (continuation of Deferred@771324440 after retry RPC after error@1314735646) -> (continuation of Deferred@616866355 after retry RPC after error@1372047058)))
21:04:29.448 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=182), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [3ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [182ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [182ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [183ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [183ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, deferred=Deferred@1727414181(state=PENDING, result=null, callback=(continuation of Deferred@354577675 after retry RPC after error@1155847070) -> (continuation of Deferred@2063561184 after retry RPC after error@757293581) -> (continuation of Deferred@737085591 after retry RPC after error@1232426583) -> (continuation of Deferred@1216630504 after retry RPC after error@2114027263) -> (continuation of Deferred@771324440 after retry RPC after error@1314735646) -> (continuation of Deferred@616866355 after retry RPC after error@1372047058), errback=(continuation of Deferred@354577675 after retry RPC after error@1155847070) -> (continuation of Deferred@2063561184 after retry RPC after error@757293581) -> (continuation of Deferred@737085591 after retry RPC after error@1232426583) -> (continuation of Deferred@1216630504 after retry RPC after error@2114027263) -> (continuation of Deferred@771324440 after retry RPC after error@1314735646) -> (continuation of Deferred@616866355 after retry RPC after error@1372047058)))
21:04:29.465 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:29.465 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:29.465 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:29.465 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:29.466 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:29.466 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:29.466 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:29.466 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:29.467 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:29.484 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:29.485 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:29.485 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:29.485 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:29.485 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:29.485 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:29.486 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:29.486 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:29.487 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:29.504 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:29.505 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:29.505 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:29.505 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:29.506 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:29.506 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:29.506 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:29.506 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:29.507 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:29.524 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:29.525 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:29.525 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:29.525 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:29.526 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:29.526 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:29.526 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:29.526 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:29.527 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:29.544 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:29.545 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:29.545 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:29.545 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:29.545 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:29.545 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:29.545 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:29.546 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:29.547 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:29.584 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:29.585 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:29.585 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:29.585 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:29.585 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:29.585 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:29.585 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:29.586 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:29.586 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:29.664 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:29.664 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:29.665 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:29.665 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:29.665 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:29.665 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:29.665 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:29.665 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:29.666 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:29.667 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=201), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [43ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [201ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [201ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [202ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, deferred=Deferred@1402124417(state=PENDING, result=null, callback=(continuation of Deferred@685402694 after retry RPC after error@546088479) -> (continuation of Deferred@2085145450 after retry RPC after error@987243218) -> (continuation of Deferred@1093396373 after retry RPC after error@72490481) -> (continuation of Deferred@1048286602 after retry RPC after error@1171189285) -> (continuation of Deferred@1531428422 after retry RPC after error@1332645194) -> (continuation of Deferred@760977128 after retry RPC after error@534182560), errback=(continuation of Deferred@685402694 after retry RPC after error@546088479) -> (continuation of Deferred@2085145450 after retry RPC after error@987243218) -> (continuation of Deferred@1093396373 after retry RPC after error@72490481) -> (continuation of Deferred@1048286602 after retry RPC after error@1171189285) -> (continuation of Deferred@1531428422 after retry RPC after error@1332645194) -> (continuation of Deferred@760977128 after retry RPC after error@534182560)))
21:04:29.667 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=201), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [43ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [201ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [201ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [202ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, deferred=Deferred@1402124417(state=PENDING, result=null, callback=(continuation of Deferred@685402694 after retry RPC after error@546088479) -> (continuation of Deferred@2085145450 after retry RPC after error@987243218) -> (continuation of Deferred@1093396373 after retry RPC after error@72490481) -> (continuation of Deferred@1048286602 after retry RPC after error@1171189285) -> (continuation of Deferred@1531428422 after retry RPC after error@1332645194) -> (continuation of Deferred@760977128 after retry RPC after error@534182560), errback=(continuation of Deferred@685402694 after retry RPC after error@546088479) -> (continuation of Deferred@2085145450 after retry RPC after error@987243218) -> (continuation of Deferred@1093396373 after retry RPC after error@72490481) -> (continuation of Deferred@1048286602 after retry RPC after error@1171189285) -> (continuation of Deferred@1531428422 after retry RPC after error@1332645194) -> (continuation of Deferred@760977128 after retry RPC after error@534182560)))
21:04:29.684 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:29.684 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:29.685 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:29.685 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:29.685 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:29.685 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:29.685 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:29.685 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:29.686 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:29.704 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:29.704 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:29.705 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:29.705 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:29.705 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:29.705 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:29.705 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:29.705 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:29.706 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:29.724 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:29.724 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:29.725 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:29.725 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:29.725 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:29.725 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:29.725 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:29.725 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:29.726 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:29.744 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:29.744 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:29.745 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:29.745 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:29.745 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:29.745 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:29.745 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:29.745 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:29.746 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:29.764 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:29.764 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:29.765 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:29.765 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:29.765 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:29.765 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:29.765 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:29.765 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:29.766 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:29.784 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:29.784 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:29.785 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:29.785 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:29.785 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:29.785 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:29.785 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:29.785 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:29.786 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:29.864 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:29.864 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:29.865 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:29.865 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:29.865 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:29.865 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:29.865 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:29.865 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:29.866 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:29.904 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:29.904 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:29.905 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:29.905 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:29.905 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:29.905 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:29.905 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:29.905 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:29.907 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:29.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:29.915135 24071 master_runner.cc:186] Error getting master registration for 127.22.168.62:37239: Network error: Client connection negotiation failed: client connection to 127.22.168.62:37239: connect: Connection refused (error 111),
21:04:29.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:29.915338 24071 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
21:04:29.944 [DEBUG - pool-40-thread-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=260), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [59ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [79ms] refreshing cache from master, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [99ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [181ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [181ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [182ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, deferred=Deferred@1688018908(state=PENDING, result=null, callback=(continuation of Deferred@824253353 after retry RPC after error@1961078328) -> (continuation of Deferred@695205310 after retry RPC after error@629175472) -> (continuation of Deferred@474341370 after retry RPC after error@1967005732) -> (continuation of Deferred@594194736 after retry RPC after error@90792985) -> (continuation of Deferred@937546049 after retry RPC after error@869984409) -> (continuation of Deferred@1162820835 after retry RPC after error@189321727) -> (continuation of Deferred@840589838 after retry RPC after error@1449188084), errback=(continuation of Deferred@824253353 after retry RPC after error@1961078328) -> (continuation of Deferred@695205310 after retry RPC after error@629175472) -> (continuation of Deferred@474341370 after retry RPC after error@1967005732) -> (continuation of Deferred@594194736 after retry RPC after error@90792985) -> (continuation of Deferred@937546049 after retry RPC after error@869984409) -> (continuation of Deferred@1162820835 after retry RPC after error@189321727) -> (continuation of Deferred@840589838 after retry RPC after error@1449188084)))
21:04:29.945 [WARN - pool-40-thread-1] (HashedWheelTimer.java:716) An exception was thrown by TimerTask.
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343) ~[main/:?]
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at java.lang.Thread.run(Thread.java:748) [?:1.8.0_275]
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86)
at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106)
at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198)
at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158)
at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953)
at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923)
at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464)
at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713)
at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701)
at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788)
at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501)
at java.lang.Thread.run(Thread.java:748)
21:04:29.964 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:29.964 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:29.965 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:29.965 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:29.965 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:29.965 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:29.965 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:29.965 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:29.966 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:29.984 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:29.985 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:29.985 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:29.985 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:29.985 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:29.985 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:29.985 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:29.985 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:29.986 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:30.004 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:30.005 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:30.005 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:30.005 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:30.005 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:30.005 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:30.005 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:30.005 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:30.006 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:30.024 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:30.024 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:30.025 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:30.025 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:30.025 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:30.025 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:30.025 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:30.025 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:30.026 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:30.044 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:30.045 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:30.045 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:30.045 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:30.045 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:30.045 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:30.045 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:30.045 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:30.046 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:30.064 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:30.065 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:30.065 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:30.065 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:30.065 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:30.065 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:30.065 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:30.066 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:30.066 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:30.125 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:30.125 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:30.125 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:30.125 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:30.125 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:30.126 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:30.126 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:30.126 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:30.127 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:30.165 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:30.165 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:30.165 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:30.165 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:30.165 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:30.166 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:30.166 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:30.166 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:30.167 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:30.167 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=203), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [162ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [162ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [163ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [163ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [201ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [202ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [202ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [203ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, deferred=Deferred@1325679702(state=PENDING, result=null, callback=(continuation of Deferred@2746771 after retry RPC after error@933349998) -> (continuation of Deferred@2060442389 after retry RPC after error@386143982) -> (continuation of Deferred@1487798508 after retry RPC after error@1642418989) -> (continuation of Deferred@1702247287 after retry RPC after error@1391653457) -> (continuation of Deferred@2062754179 after retry RPC after error@834955723) -> (continuation of Deferred@1698513159 after retry RPC after error@538466187) -> (continuation of Deferred@1124373810 after retry RPC after error@1369192104), errback=(continuation of Deferred@2746771 after retry RPC after error@933349998) -> (continuation of Deferred@2060442389 after retry RPC after error@386143982) -> (continuation of Deferred@1487798508 after retry RPC after error@1642418989) -> (continuation of Deferred@1702247287 after retry RPC after error@1391653457) -> (continuation of Deferred@2062754179 after retry RPC after error@834955723) -> (continuation of Deferred@1698513159 after retry RPC after error@538466187) -> (continuation of Deferred@1124373810 after retry RPC after error@1369192104)))
21:04:30.167 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=203), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [162ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [162ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [163ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [163ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [201ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [202ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [202ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [203ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, deferred=Deferred@1325679702(state=PENDING, result=null, callback=(continuation of Deferred@2746771 after retry RPC after error@933349998) -> (continuation of Deferred@2060442389 after retry RPC after error@386143982) -> (continuation of Deferred@1487798508 after retry RPC after error@1642418989) -> (continuation of Deferred@1702247287 after retry RPC after error@1391653457) -> (continuation of Deferred@2062754179 after retry RPC after error@834955723) -> (continuation of Deferred@1698513159 after retry RPC after error@538466187) -> (continuation of Deferred@1124373810 after retry RPC after error@1369192104), errback=(continuation of Deferred@2746771 after retry RPC after error@933349998) -> (continuation of Deferred@2060442389 after retry RPC after error@386143982) -> (continuation of Deferred@1487798508 after retry RPC after error@1642418989) -> (continuation of Deferred@1702247287 after retry RPC after error@1391653457) -> (continuation of Deferred@2062754179 after retry RPC after error@834955723) -> (continuation of Deferred@1698513159 after retry RPC after error@538466187) -> (continuation of Deferred@1124373810 after retry RPC after error@1369192104)))
21:04:30.184 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:30.185 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:30.185 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:30.186 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:30.186 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:30.186 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:30.186 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:30.186 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:30.187 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:30.204 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:30.205 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:30.205 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:30.205 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:30.206 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:30.206 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:30.206 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:30.206 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:30.207 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:30.224 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:30.225 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:30.225 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:30.226 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:30.226 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:30.226 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:30.226 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:30.226 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:30.227 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:30.244 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:30.245 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:30.245 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:30.245 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:30.245 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:30.246 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:30.246 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:30.246 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:30.247 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:30.265 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:30.265 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:30.266 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:30.266 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:30.266 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:30.266 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:30.266 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:30.266 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:30.267 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:30.285 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:30.285 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:30.285 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:30.285 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:30.285 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:30.286 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:30.286 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:30.286 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:30.287 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:30.344 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:30.344 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:30.345 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:30.345 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:30.345 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:30.345 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:30.345 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:30.345 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:30.346 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:30.365 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:30.365 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:30.366 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:30.366 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:30.366 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:30.366 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:30.366 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:30.366 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:30.367 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:30.368 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=182), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [3ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [43ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [83ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [103ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [159ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [161ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [161ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [162ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [182ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [182ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [183ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [183ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, deferred=Deferred@453194055(state=PENDING, result=null, callback=(continuation of Deferred@507226557 after retry RPC after error@751039679) -> (continuation of Deferred@2139807817 after retry RPC after error@2097250294) -> (continuation of Deferred@1043617706 after retry RPC after error@224471775) -> (continuation of Deferred@625023890 after retry RPC after error@1397304835) -> (continuation of Deferred@1339865799 after retry RPC after error@1265583665) -> (continuation of Deferred@1725913413 after retry RPC after error@2088524459) -> (continuation of Deferred@1448922465 after retry RPC after error@1731413845), errback=(continuation of Deferred@507226557 after retry RPC after error@751039679) -> (continuation of Deferred@2139807817 after retry RPC after error@2097250294) -> (continuation of Deferred@1043617706 after retry RPC after error@224471775) -> (continuation of Deferred@625023890 after retry RPC after error@1397304835) -> (continuation of Deferred@1339865799 after retry RPC after error@1265583665) -> (continuation of Deferred@1725913413 after retry RPC after error@2088524459) -> (continuation of Deferred@1448922465 after retry RPC after error@1731413845)))
21:04:30.368 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=182), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [3ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [43ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [83ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [103ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [159ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [161ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [161ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [162ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [182ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [182ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [183ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [183ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, deferred=Deferred@453194055(state=PENDING, result=null, callback=(continuation of Deferred@507226557 after retry RPC after error@751039679) -> (continuation of Deferred@2139807817 after retry RPC after error@2097250294) -> (continuation of Deferred@1043617706 after retry RPC after error@224471775) -> (continuation of Deferred@625023890 after retry RPC after error@1397304835) -> (continuation of Deferred@1339865799 after retry RPC after error@1265583665) -> (continuation of Deferred@1725913413 after retry RPC after error@2088524459) -> (continuation of Deferred@1448922465 after retry RPC after error@1731413845), errback=(continuation of Deferred@507226557 after retry RPC after error@751039679) -> (continuation of Deferred@2139807817 after retry RPC after error@2097250294) -> (continuation of Deferred@1043617706 after retry RPC after error@224471775) -> (continuation of Deferred@625023890 after retry RPC after error@1397304835) -> (continuation of Deferred@1339865799 after retry RPC after error@1265583665) -> (continuation of Deferred@1725913413 after retry RPC after error@2088524459) -> (continuation of Deferred@1448922465 after retry RPC after error@1731413845)))
21:04:30.385 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:30.385 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:30.385 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:30.385 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:30.386 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:30.386 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:30.386 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:30.386 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:30.387 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:30.404 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:30.405 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:30.405 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:30.405 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:30.406 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:30.406 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:30.406 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:30.406 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:30.406 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:30.424 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:30.425 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:30.425 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:30.425 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:30.425 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:30.425 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:30.426 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:30.426 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:30.426 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:30.444 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:30.445 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:30.445 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:30.445 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:30.445 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:30.445 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:30.445 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:30.445 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:30.446 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:30.464 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:30.465 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:30.465 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:30.465 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:30.465 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:30.465 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:30.465 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:30.465 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:30.466 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:30.484 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:30.485 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:30.485 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:30.485 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:30.485 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:30.485 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:30.485 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:30.485 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:30.486 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:30.544 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:30.545 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:30.545 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:30.545 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:30.545 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:30.545 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:30.545 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:30.546 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:30.546 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:30.585 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:30.585 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:30.586 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:30.586 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:30.586 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:30.586 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:30.586 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:30.586 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:30.587 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:30.587 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=202), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [161ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [161ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [162ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [163ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [202ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [202ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [203ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [203ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, deferred=Deferred@1507718976(state=PENDING, result=null, callback=(continuation of Deferred@1755619278 after retry RPC after error@441194143) -> (continuation of Deferred@3464554 after retry RPC after error@920997339) -> (continuation of Deferred@2062806566 after retry RPC after error@7464713) -> (continuation of Deferred@638100960 after retry RPC after error@975159151) -> (continuation of Deferred@1610151124 after retry RPC after error@1354631565) -> (continuation of Deferred@872059359 after retry RPC after error@1952031752) -> (continuation of Deferred@1814016003 after retry RPC after error@1288519276), errback=(continuation of Deferred@1755619278 after retry RPC after error@441194143) -> (continuation of Deferred@3464554 after retry RPC after error@920997339) -> (continuation of Deferred@2062806566 after retry RPC after error@7464713) -> (continuation of Deferred@638100960 after retry RPC after error@975159151) -> (continuation of Deferred@1610151124 after retry RPC after error@1354631565) -> (continuation of Deferred@872059359 after retry RPC after error@1952031752) -> (continuation of Deferred@1814016003 after retry RPC after error@1288519276)))
21:04:30.588 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=202), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [161ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [161ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [162ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [163ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [202ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [202ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [203ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [203ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, deferred=Deferred@1507718976(state=PENDING, result=null, callback=(continuation of Deferred@1755619278 after retry RPC after error@441194143) -> (continuation of Deferred@3464554 after retry RPC after error@920997339) -> (continuation of Deferred@2062806566 after retry RPC after error@7464713) -> (continuation of Deferred@638100960 after retry RPC after error@975159151) -> (continuation of Deferred@1610151124 after retry RPC after error@1354631565) -> (continuation of Deferred@872059359 after retry RPC after error@1952031752) -> (continuation of Deferred@1814016003 after retry RPC after error@1288519276), errback=(continuation of Deferred@1755619278 after retry RPC after error@441194143) -> (continuation of Deferred@3464554 after retry RPC after error@920997339) -> (continuation of Deferred@2062806566 after retry RPC after error@7464713) -> (continuation of Deferred@638100960 after retry RPC after error@975159151) -> (continuation of Deferred@1610151124 after retry RPC after error@1354631565) -> (continuation of Deferred@872059359 after retry RPC after error@1952031752) -> (continuation of Deferred@1814016003 after retry RPC after error@1288519276)))
21:04:30.604 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:30.604 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:30.605 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:30.605 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:30.605 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:30.605 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:30.605 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:30.605 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:30.606 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:30.625 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:30.625 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:30.626 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:30.626 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:30.626 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:30.626 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:30.626 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:30.626 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:30.627 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:30.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:30.633531 24327 raft_consensus.cc:493] T 00000000000000000000000000000000 P eb664add87f44013b596f4cc01c9a41c [term 4 FOLLOWER]: Starting pre-election (detected failure of leader c520077acf3341e29447c6c2b8d3f74c)
21:04:30.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:30.633749 24327 raft_consensus.cc:515] T 00000000000000000000000000000000 P eb664add87f44013b596f4cc01c9a41c [term 4 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e9dd218ca2244f5a934aac2b7e58eb1d" member_type: VOTER last_known_addr { host: "127.22.168.62" port: 37239 } } peers { permanent_uuid: "c520077acf3341e29447c6c2b8d3f74c" member_type: VOTER last_known_addr { host: "127.22.168.61" port: 32947 } } peers { permanent_uuid: "eb664add87f44013b596f4cc01c9a41c" member_type: VOTER last_known_addr { host: "127.22.168.60" port: 33247 } }
21:04:30.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:30.634454 24327 leader_election.cc:290] T 00000000000000000000000000000000 P eb664add87f44013b596f4cc01c9a41c [CANDIDATE]: Term 5 pre-election: Requested pre-vote from peers e9dd218ca2244f5a934aac2b7e58eb1d (127.22.168.62:37239), c520077acf3341e29447c6c2b8d3f74c (127.22.168.61:32947)
21:04:30.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:30.636698 24089 leader_election.cc:336] T 00000000000000000000000000000000 P eb664add87f44013b596f4cc01c9a41c [CANDIDATE]: Term 5 pre-election: RPC error from VoteRequest() call to peer c520077acf3341e29447c6c2b8d3f74c (127.22.168.61:32947): Network error: Client connection negotiation failed: client connection to 127.22.168.61:32947: connect: Connection refused (error 111)
21:04:30.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:30.636869 24086 leader_election.cc:336] T 00000000000000000000000000000000 P eb664add87f44013b596f4cc01c9a41c [CANDIDATE]: Term 5 pre-election: RPC error from VoteRequest() call to peer e9dd218ca2244f5a934aac2b7e58eb1d (127.22.168.62:37239): Network error: Client connection negotiation failed: client connection to 127.22.168.62:37239: connect: Connection refused (error 111)
21:04:30.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:30.637004 24086 leader_election.cc:304] T 00000000000000000000000000000000 P eb664add87f44013b596f4cc01c9a41c [CANDIDATE]: Term 5 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: eb664add87f44013b596f4cc01c9a41c; no voters: c520077acf3341e29447c6c2b8d3f74c, e9dd218ca2244f5a934aac2b7e58eb1d
21:04:30.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:30.637310 24327 raft_consensus.cc:2749] T 00000000000000000000000000000000 P eb664add87f44013b596f4cc01c9a41c [term 4 FOLLOWER]: Leader pre-election lost for term 5. Reason: could not achieve majority
21:04:30.644 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:30.644 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:30.645 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:30.645 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:30.645 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:30.645 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:30.645 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:30.645 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:30.646 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:30.665 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:30.665 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:30.665 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:30.665 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:30.666 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:30.666 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:30.666 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:30.666 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:30.666 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:30.684 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:30.685 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:30.685 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:30.686 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:30.686 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:30.686 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:30.686 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:30.686 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:30.686 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:30.704 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:30.705 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:30.705 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:30.705 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:30.705 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:30.705 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:30.705 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:30.706 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:30.706 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:30.765 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:30.765 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:30.765 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:30.766 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:30.766 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:30.766 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:30.766 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:30.766 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:30.767 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:30.864 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:30.865 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:30.865 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:30.865 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:30.865 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:30.865 [DEBUG - pool-40-thread-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=261), Traces: [0ms] refreshing cache from master, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [22ms] refreshing cache from master, [22ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [22ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [22ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [23ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [23ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [23ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [24ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [40ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [61ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [62ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [62ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [63ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [63ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [63ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [81ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [82ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [82ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [83ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [83ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [83ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [84ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [101ms] refreshing cache from master, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [102ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [102ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [103ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [104ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [161ms] refreshing cache from master, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [162ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [162ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [163ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [163ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [163ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [164ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, deferred=Deferred@1393739968(state=PENDING, result=null, callback=(continuation of Deferred@757651093 after retry RPC after error@117331239) -> (continuation of Deferred@1183395303 after retry RPC after error@51947838) -> (continuation of Deferred@1760512950 after retry RPC after error@483163361) -> (continuation of Deferred@329616522 after retry RPC after error@1429378713) -> (continuation of Deferred@1200805234 after retry RPC after error@952533525) -> (continuation of Deferred@129538139 after retry RPC after error@1191137387) -> (continuation of Deferred@387392643 after retry RPC after error@466471518), errback=(continuation of Deferred@757651093 after retry RPC after error@117331239) -> (continuation of Deferred@1183395303 after retry RPC after error@51947838) -> (continuation of Deferred@1760512950 after retry RPC after error@483163361) -> (continuation of Deferred@329616522 after retry RPC after error@1429378713) -> (continuation of Deferred@1200805234 after retry RPC after error@952533525) -> (continuation of Deferred@129538139 after retry RPC after error@1191137387) -> (continuation of Deferred@387392643 after retry RPC after error@466471518)))
21:04:30.865 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:30.865 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:30.866 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:30.866 [WARN - pool-40-thread-1] (HashedWheelTimer.java:716) An exception was thrown by TimerTask.
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343) ~[main/:?]
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at java.lang.Thread.run(Thread.java:748) [?:1.8.0_275]
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86)
at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106)
at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198)
at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158)
at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953)
at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923)
at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464)
at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713)
at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701)
at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788)
at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501)
at java.lang.Thread.run(Thread.java:748)
21:04:30.867 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:30.884 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:30.884 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:30.885 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:30.885 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:30.885 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:30.885 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:30.885 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:30.885 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:30.886 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:30.904 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:30.905 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:30.905 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:30.905 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:30.905 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:30.905 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:30.905 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:30.905 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:30.906 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:30.924 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:30.925 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:30.925 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:30.925 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:30.925 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:30.925 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:30.926 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:30.926 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:30.926 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:30.944 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:30.945 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:30.945 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:30.945 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:30.945 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:30.945 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:30.945 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:30.945 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:30.947 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:30.964 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:30.965 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:30.965 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:30.965 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:30.965 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:30.965 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:30.965 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:30.965 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:30.966 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:30.984 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:30.985 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:30.985 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:30.985 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:30.985 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:30.985 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:30.985 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:30.985 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:30.986 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:31.024 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:31.025 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:31.025 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:31.025 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:31.025 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:31.025 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:31.026 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:31.026 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:31.027 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:31.104 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:31.104 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:31.105 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:31.105 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:31.105 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:31.105 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:31.105 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:31.105 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:31.106 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:31.106 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=222), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [141ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [142ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [142ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [143ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [221ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [221ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [222ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [222ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, deferred=Deferred@2021550017(state=PENDING, result=null, callback=(continuation of Deferred@1989147643 after retry RPC after error@161865905) -> (continuation of Deferred@460334417 after retry RPC after error@280693137) -> (continuation of Deferred@434268254 after retry RPC after error@1071952827) -> (continuation of Deferred@623133627 after retry RPC after error@1838883087) -> (continuation of Deferred@1739654266 after retry RPC after error@2110921308) -> (continuation of Deferred@2053945286 after retry RPC after error@1321951965) -> (continuation of Deferred@565591466 after retry RPC after error@1281082170), errback=(continuation of Deferred@1989147643 after retry RPC after error@161865905) -> (continuation of Deferred@460334417 after retry RPC after error@280693137) -> (continuation of Deferred@434268254 after retry RPC after error@1071952827) -> (continuation of Deferred@623133627 after retry RPC after error@1838883087) -> (continuation of Deferred@1739654266 after retry RPC after error@2110921308) -> (continuation of Deferred@2053945286 after retry RPC after error@1321951965) -> (continuation of Deferred@565591466 after retry RPC after error@1281082170)))
21:04:31.107 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=222), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [141ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [142ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [142ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [143ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [221ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [221ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [222ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [222ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, deferred=Deferred@2021550017(state=PENDING, result=null, callback=(continuation of Deferred@1989147643 after retry RPC after error@161865905) -> (continuation of Deferred@460334417 after retry RPC after error@280693137) -> (continuation of Deferred@434268254 after retry RPC after error@1071952827) -> (continuation of Deferred@623133627 after retry RPC after error@1838883087) -> (continuation of Deferred@1739654266 after retry RPC after error@2110921308) -> (continuation of Deferred@2053945286 after retry RPC after error@1321951965) -> (continuation of Deferred@565591466 after retry RPC after error@1281082170), errback=(continuation of Deferred@1989147643 after retry RPC after error@161865905) -> (continuation of Deferred@460334417 after retry RPC after error@280693137) -> (continuation of Deferred@434268254 after retry RPC after error@1071952827) -> (continuation of Deferred@623133627 after retry RPC after error@1838883087) -> (continuation of Deferred@1739654266 after retry RPC after error@2110921308) -> (continuation of Deferred@2053945286 after retry RPC after error@1321951965) -> (continuation of Deferred@565591466 after retry RPC after error@1281082170)))
21:04:31.124 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:31.124 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:31.125 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:31.125 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:31.125 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:31.125 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:31.125 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:31.125 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:31.126 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:31.144 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:31.144 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:31.145 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:31.145 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:31.145 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:31.145 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:31.145 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:31.145 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:31.146 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:31.164 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:31.164 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:31.165 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:31.165 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:31.165 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:31.165 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:31.165 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:31.165 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:31.166 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:31.184 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:31.184 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:31.185 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:31.185 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:31.185 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:31.185 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:31.185 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:31.185 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:31.186 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:31.204 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:31.204 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:31.205 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:31.205 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:31.205 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:31.205 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:31.205 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:31.205 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:31.206 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:31.224 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:31.224 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:31.225 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:31.225 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:31.225 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:31.225 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:31.225 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:31.225 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:31.226 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:31.264 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:31.265 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:31.265 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:31.265 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:31.265 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:31.265 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:31.265 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:31.265 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:31.267 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:31.304 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:31.305 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:31.305 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:31.305 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:31.305 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:31.306 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:31.306 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:31.306 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:31.307 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:31.325 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:31.325 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:31.326 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:31.326 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:31.326 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:31.326 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:31.327 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:31.327 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:31.327 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:31.327 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=9, TimeoutTracker(timeout=250, elapsed=203), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [79ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [99ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [141ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [141ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [142ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [143ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [181ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [182ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [183ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [183ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [202ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [203ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [203ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [203ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, deferred=Deferred@420037742(state=PENDING, result=null, callback=(continuation of Deferred@965379274 after retry RPC after error@719980616) -> (continuation of Deferred@1094747470 after retry RPC after error@305759036) -> (continuation of Deferred@2147019109 after retry RPC after error@86644378) -> (continuation of Deferred@1082497412 after retry RPC after error@519819979) -> (continuation of Deferred@644916144 after retry RPC after error@1933311778) -> (continuation of Deferred@522064248 after retry RPC after error@1605625306) -> (continuation of Deferred@2088607935 after retry RPC after error@1351848701) -> (continuation of Deferred@1063898182 after retry RPC after error@2134143812), errback=(continuation of Deferred@965379274 after retry RPC after error@719980616) -> (continuation of Deferred@1094747470 after retry RPC after error@305759036) -> (continuation of Deferred@2147019109 after retry RPC after error@86644378) -> (continuation of Deferred@1082497412 after retry RPC after error@519819979) -> (continuation of Deferred@644916144 after retry RPC after error@1933311778) -> (continuation of Deferred@522064248 after retry RPC after error@1605625306) -> (continuation of Deferred@2088607935 after retry RPC after error@1351848701) -> (continuation of Deferred@1063898182 after retry RPC after error@2134143812)))
21:04:31.328 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=9, TimeoutTracker(timeout=250, elapsed=203), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [79ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [99ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [141ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [141ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [142ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [143ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [181ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [182ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [183ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [183ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [202ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [203ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [203ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [203ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, deferred=Deferred@420037742(state=PENDING, result=null, callback=(continuation of Deferred@965379274 after retry RPC after error@719980616) -> (continuation of Deferred@1094747470 after retry RPC after error@305759036) -> (continuation of Deferred@2147019109 after retry RPC after error@86644378) -> (continuation of Deferred@1082497412 after retry RPC after error@519819979) -> (continuation of Deferred@644916144 after retry RPC after error@1933311778) -> (continuation of Deferred@522064248 after retry RPC after error@1605625306) -> (continuation of Deferred@2088607935 after retry RPC after error@1351848701) -> (continuation of Deferred@1063898182 after retry RPC after error@2134143812), errback=(continuation of Deferred@965379274 after retry RPC after error@719980616) -> (continuation of Deferred@1094747470 after retry RPC after error@305759036) -> (continuation of Deferred@2147019109 after retry RPC after error@86644378) -> (continuation of Deferred@1082497412 after retry RPC after error@519819979) -> (continuation of Deferred@644916144 after retry RPC after error@1933311778) -> (continuation of Deferred@522064248 after retry RPC after error@1605625306) -> (continuation of Deferred@2088607935 after retry RPC after error@1351848701) -> (continuation of Deferred@1063898182 after retry RPC after error@2134143812)))
21:04:31.344 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:31.345 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:31.345 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:31.345 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:31.345 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:31.345 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:31.346 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:31.346 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:31.347 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:31.364 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:31.365 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:31.365 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:31.365 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:31.365 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:31.366 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:31.366 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:31.366 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:31.367 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:31.384 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:31.385 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:31.385 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:31.385 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:31.385 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:31.385 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:31.385 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:31.385 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:31.386 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:31.404 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:31.404 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:31.405 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:31.405 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:31.405 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:31.405 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:31.405 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:31.405 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:31.406 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:31.424 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:31.424 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:31.425 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:31.425 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:31.425 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:31.425 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:31.425 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:31.425 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:31.426 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:31.444 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:31.445 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:31.445 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:31.445 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:31.445 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:31.445 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:31.445 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:31.445 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:31.446 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:31.484 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:31.485 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:31.485 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:31.485 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:31.485 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:31.485 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:31.485 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:31.485 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:31.486 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:31.584 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:31.584 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:31.585 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:31.585 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:31.585 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:31.585 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:31.585 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:31.585 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:31.586 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:31.586 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=241), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [141ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [141ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [142ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [239ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [241ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [241ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [242ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [242ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, deferred=Deferred@1797379469(state=PENDING, result=null, callback=(continuation of Deferred@1821376583 after retry RPC after error@513369019) -> (continuation of Deferred@1450671041 after retry RPC after error@866079039) -> (continuation of Deferred@1600253235 after retry RPC after error@179358175) -> (continuation of Deferred@1735655995 after retry RPC after error@774611267) -> (continuation of Deferred@2129906112 after retry RPC after error@1912670704) -> (continuation of Deferred@891533393 after retry RPC after error@1190209039) -> (continuation of Deferred@648825560 after retry RPC after error@1553500532), errback=(continuation of Deferred@1821376583 after retry RPC after error@513369019) -> (continuation of Deferred@1450671041 after retry RPC after error@866079039) -> (continuation of Deferred@1600253235 after retry RPC after error@179358175) -> (continuation of Deferred@1735655995 after retry RPC after error@774611267) -> (continuation of Deferred@2129906112 after retry RPC after error@1912670704) -> (continuation of Deferred@891533393 after retry RPC after error@1190209039) -> (continuation of Deferred@648825560 after retry RPC after error@1553500532)))
21:04:31.587 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=241), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [141ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [141ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [142ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [239ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [241ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [241ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [242ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [242ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, deferred=Deferred@1797379469(state=PENDING, result=null, callback=(continuation of Deferred@1821376583 after retry RPC after error@513369019) -> (continuation of Deferred@1450671041 after retry RPC after error@866079039) -> (continuation of Deferred@1600253235 after retry RPC after error@179358175) -> (continuation of Deferred@1735655995 after retry RPC after error@774611267) -> (continuation of Deferred@2129906112 after retry RPC after error@1912670704) -> (continuation of Deferred@891533393 after retry RPC after error@1190209039) -> (continuation of Deferred@648825560 after retry RPC after error@1553500532), errback=(continuation of Deferred@1821376583 after retry RPC after error@513369019) -> (continuation of Deferred@1450671041 after retry RPC after error@866079039) -> (continuation of Deferred@1600253235 after retry RPC after error@179358175) -> (continuation of Deferred@1735655995 after retry RPC after error@774611267) -> (continuation of Deferred@2129906112 after retry RPC after error@1912670704) -> (continuation of Deferred@891533393 after retry RPC after error@1190209039) -> (continuation of Deferred@648825560 after retry RPC after error@1553500532)))
21:04:31.604 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:31.604 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:31.605 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:31.605 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:31.605 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:31.605 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:31.605 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:31.605 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:31.606 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:31.624 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:31.624 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:31.625 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:31.625 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:31.625 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:31.625 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:31.625 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:31.625 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:31.626 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:31.644 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:31.644 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:31.644 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:31.644 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:31.645 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:31.645 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:31.645 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:31.645 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:31.646 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:31.665 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:31.665 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:31.665 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:31.665 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:31.666 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:31.666 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:31.666 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:31.666 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:31.667 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:31.684 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:31.685 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:31.685 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:31.685 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:31.685 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:31.685 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:31.686 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:31.686 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:31.687 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:31.724 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:31.725 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:31.725 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:31.725 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:31.726 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:31.726 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:31.726 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:31.726 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:31.727 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:31.744 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:31.745 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:31.745 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:31.745 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:31.746 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:31.746 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:31.746 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:31.746 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:31.747 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:31.844 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:31.844 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:31.845 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:31.845 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:31.845 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:31.845 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:31.845 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:31.845 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:31.846 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:31.847 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=242), Traces: [0ms] refreshing cache from master, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [3ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [21ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [23ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [40ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [43ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [61ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [62ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [62ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [63ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [63ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [63ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [64ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [81ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [82ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [82ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [83ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [83ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [84ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [121ms] refreshing cache from master, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [122ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [122ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [123ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [123ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [123ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [124ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [141ms] refreshing cache from master, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [142ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [142ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [143ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [143ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [144ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [144ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [241ms] refreshing cache from master, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [242ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [242ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [243ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [243ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, deferred=Deferred@1498695088(state=PENDING, result=null, callback=(continuation of Deferred@129012279 after retry RPC after error@2042654044) -> (continuation of Deferred@79578684 after retry RPC after error@2143091422) -> (continuation of Deferred@2090794548 after retry RPC after error@114218614) -> (continuation of Deferred@1891102517 after retry RPC after error@1749385719) -> (continuation of Deferred@1671540117 after retry RPC after error@321698545) -> (continuation of Deferred@1546461670 after retry RPC after error@413358509) -> (continuation of Deferred@1979744809 after retry RPC after error@873770), errback=(continuation of Deferred@129012279 after retry RPC after error@2042654044) -> (continuation of Deferred@79578684 after retry RPC after error@2143091422) -> (continuation of Deferred@2090794548 after retry RPC after error@114218614) -> (continuation of Deferred@1891102517 after retry RPC after error@1749385719) -> (continuation of Deferred@1671540117 after retry RPC after error@321698545) -> (continuation of Deferred@1546461670 after retry RPC after error@413358509) -> (continuation of Deferred@1979744809 after retry RPC after error@873770)))
21:04:31.847 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=242), Traces: [0ms] refreshing cache from master, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [3ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [21ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [23ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [40ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [43ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [61ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [62ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [62ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [63ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [63ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [63ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [64ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [81ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [82ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [82ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [83ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [83ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [84ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [121ms] refreshing cache from master, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [122ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [122ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [123ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [123ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [123ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [124ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [141ms] refreshing cache from master, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [142ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [142ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [143ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [143ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [144ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [144ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [241ms] refreshing cache from master, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [242ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [242ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [243ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [243ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, deferred=Deferred@1498695088(state=PENDING, result=null, callback=(continuation of Deferred@129012279 after retry RPC after error@2042654044) -> (continuation of Deferred@79578684 after retry RPC after error@2143091422) -> (continuation of Deferred@2090794548 after retry RPC after error@114218614) -> (continuation of Deferred@1891102517 after retry RPC after error@1749385719) -> (continuation of Deferred@1671540117 after retry RPC after error@321698545) -> (continuation of Deferred@1546461670 after retry RPC after error@413358509) -> (continuation of Deferred@1979744809 after retry RPC after error@873770), errback=(continuation of Deferred@129012279 after retry RPC after error@2042654044) -> (continuation of Deferred@79578684 after retry RPC after error@2143091422) -> (continuation of Deferred@2090794548 after retry RPC after error@114218614) -> (continuation of Deferred@1891102517 after retry RPC after error@1749385719) -> (continuation of Deferred@1671540117 after retry RPC after error@321698545) -> (continuation of Deferred@1546461670 after retry RPC after error@413358509) -> (continuation of Deferred@1979744809 after retry RPC after error@873770)))
21:04:31.864 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:31.864 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:31.865 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:31.865 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:31.865 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:31.865 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:31.865 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:31.865 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:31.866 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:31.884 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:31.884 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:31.885 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:31.885 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:31.885 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:31.885 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:31.885 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:31.885 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:31.886 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:31.904 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:31.904 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:31.905 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:31.905 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:31.905 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:31.905 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:31.905 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:31.905 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:31.906 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:31.924 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:31.924 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:31.925 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:31.925 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:31.925 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:31.925 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:31.925 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:31.925 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:31.926 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:31.944 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:31.945 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:31.945 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:31.945 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:31.945 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:31.945 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:31.945 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:31.945 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:31.946 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:31.964 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:31.965 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:31.965 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:31.965 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:31.965 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:31.965 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:31.965 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:31.965 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:31.966 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:32.004 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:32.005 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:32.005 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:32.005 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:32.006 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:32.006 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:32.006 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:32.006 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:32.007 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:32.124 [DEBUG - pool-40-thread-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=260), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [142ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [142ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [142ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [143ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, deferred=Deferred@849835951(state=PENDING, result=null, callback=(continuation of Deferred@1014193779 after retry RPC after error@1827965886) -> (continuation of Deferred@1585649834 after retry RPC after error@1469060826) -> (continuation of Deferred@1907772061 after retry RPC after error@962041454) -> (continuation of Deferred@2144837542 after retry RPC after error@681992256) -> (continuation of Deferred@1804023147 after retry RPC after error@1860582035) -> (continuation of Deferred@764387910 after retry RPC after error@917533655) -> (continuation of Deferred@1691235335 after retry RPC after error@992192687), errback=(continuation of Deferred@1014193779 after retry RPC after error@1827965886) -> (continuation of Deferred@1585649834 after retry RPC after error@1469060826) -> (continuation of Deferred@1907772061 after retry RPC after error@962041454) -> (continuation of Deferred@2144837542 after retry RPC after error@681992256) -> (continuation of Deferred@1804023147 after retry RPC after error@1860582035) -> (continuation of Deferred@764387910 after retry RPC after error@917533655) -> (continuation of Deferred@1691235335 after retry RPC after error@992192687)))
21:04:32.125 [WARN - pool-40-thread-1] (HashedWheelTimer.java:716) An exception was thrown by TimerTask.
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343) ~[main/:?]
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at java.lang.Thread.run(Thread.java:748) [?:1.8.0_275]
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86)
at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106)
at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198)
at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158)
at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953)
at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923)
at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464)
at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713)
at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701)
at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788)
at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501)
at java.lang.Thread.run(Thread.java:748)
21:04:32.144 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:32.144 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:32.145 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:32.145 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:32.145 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:32.145 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:32.145 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:32.145 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:32.146 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:32.164 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:32.164 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:32.165 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:32.165 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:32.165 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:32.165 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:32.165 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:32.165 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:32.166 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:32.184 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:32.184 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:32.185 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:32.185 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:32.185 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:32.185 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:32.185 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:32.186 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:32.186 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:32.204 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:32.204 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:32.205 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:32.205 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:32.205 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:32.205 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:32.205 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:32.205 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:32.206 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:32.224 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:32.224 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:32.225 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:32.225 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:32.225 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:32.226 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:32.226 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:32.226 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:32.227 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:32.264 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:32.265 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:32.265 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:32.265 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:32.265 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:32.265 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:32.266 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:32.266 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:32.266 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:32.325 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:32.325 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:32.325 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:32.326 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:32.327 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:32.325 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:32.327 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:32.327 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:32.327 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:32.365 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:32.365 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:32.365 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:32.365 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:32.366 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:32.366 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:32.366 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:32.366 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:32.366 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:32.367 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=222), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [83ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [182ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [183ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [183ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [183ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [222ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [222ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [222ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [223ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, deferred=Deferred@1043493281(state=PENDING, result=null, callback=(continuation of Deferred@1726648247 after retry RPC after error@252579142) -> (continuation of Deferred@106905238 after retry RPC after error@734918789) -> (continuation of Deferred@426270076 after retry RPC after error@1718814506) -> (continuation of Deferred@421998332 after retry RPC after error@1567286413) -> (continuation of Deferred@112378337 after retry RPC after error@2013711434) -> (continuation of Deferred@1492540792 after retry RPC after error@1379042407) -> (continuation of Deferred@1117780489 after retry RPC after error@213191308), errback=(continuation of Deferred@1726648247 after retry RPC after error@252579142) -> (continuation of Deferred@106905238 after retry RPC after error@734918789) -> (continuation of Deferred@426270076 after retry RPC after error@1718814506) -> (continuation of Deferred@421998332 after retry RPC after error@1567286413) -> (continuation of Deferred@112378337 after retry RPC after error@2013711434) -> (continuation of Deferred@1492540792 after retry RPC after error@1379042407) -> (continuation of Deferred@1117780489 after retry RPC after error@213191308)))
21:04:32.367 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=222), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [83ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [182ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [183ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [183ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [183ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [222ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [222ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [222ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [223ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, deferred=Deferred@1043493281(state=PENDING, result=null, callback=(continuation of Deferred@1726648247 after retry RPC after error@252579142) -> (continuation of Deferred@106905238 after retry RPC after error@734918789) -> (continuation of Deferred@426270076 after retry RPC after error@1718814506) -> (continuation of Deferred@421998332 after retry RPC after error@1567286413) -> (continuation of Deferred@112378337 after retry RPC after error@2013711434) -> (continuation of Deferred@1492540792 after retry RPC after error@1379042407) -> (continuation of Deferred@1117780489 after retry RPC after error@213191308), errback=(continuation of Deferred@1726648247 after retry RPC after error@252579142) -> (continuation of Deferred@106905238 after retry RPC after error@734918789) -> (continuation of Deferred@426270076 after retry RPC after error@1718814506) -> (continuation of Deferred@421998332 after retry RPC after error@1567286413) -> (continuation of Deferred@112378337 after retry RPC after error@2013711434) -> (continuation of Deferred@1492540792 after retry RPC after error@1379042407) -> (continuation of Deferred@1117780489 after retry RPC after error@213191308)))
21:04:32.384 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:32.385 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:32.385 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:32.385 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:32.385 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:32.386 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:32.386 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:32.386 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:32.387 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:32.404 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:32.405 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:32.405 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:32.405 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:32.405 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:32.405 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:32.406 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:32.406 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:32.407 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:32.424 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:32.425 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:32.425 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:32.425 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:32.425 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:32.425 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:32.426 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:32.426 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:32.426 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:32.444 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:32.445 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:32.445 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:32.445 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:32.445 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:32.445 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:32.445 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:32.445 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:32.446 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:32.464 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:32.465 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:32.465 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:32.465 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:32.465 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:32.465 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:32.465 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:32.465 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:32.466 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:32.504 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:32.505 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:32.505 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:32.505 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:32.505 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:32.505 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:32.505 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:32.506 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:32.506 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:32.584 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:32.584 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:32.585 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:32.585 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:32.585 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:32.585 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:32.585 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:32.585 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:32.586 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:32.604 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:32.604 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:32.605 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:32.605 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:32.605 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:32.605 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:32.605 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:32.605 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:32.606 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:32.607 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=222), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [23ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [199ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [201ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [201ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [202ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [221ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [221ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [222ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [222ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, deferred=Deferred@1981329269(state=PENDING, result=null, callback=(continuation of Deferred@1924179696 after retry RPC after error@1267320584) -> (continuation of Deferred@1245290720 after retry RPC after error@2045369813) -> (continuation of Deferred@244937534 after retry RPC after error@187520098) -> (continuation of Deferred@202693447 after retry RPC after error@992262245) -> (continuation of Deferred@2120297660 after retry RPC after error@521075030) -> (continuation of Deferred@904582931 after retry RPC after error@731096667) -> (continuation of Deferred@1549826905 after retry RPC after error@885849546), errback=(continuation of Deferred@1924179696 after retry RPC after error@1267320584) -> (continuation of Deferred@1245290720 after retry RPC after error@2045369813) -> (continuation of Deferred@244937534 after retry RPC after error@187520098) -> (continuation of Deferred@202693447 after retry RPC after error@992262245) -> (continuation of Deferred@2120297660 after retry RPC after error@521075030) -> (continuation of Deferred@904582931 after retry RPC after error@731096667) -> (continuation of Deferred@1549826905 after retry RPC after error@885849546)))
21:04:32.607 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=222), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [23ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [199ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [201ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [201ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [202ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [221ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [221ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [222ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [222ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, deferred=Deferred@1981329269(state=PENDING, result=null, callback=(continuation of Deferred@1924179696 after retry RPC after error@1267320584) -> (continuation of Deferred@1245290720 after retry RPC after error@2045369813) -> (continuation of Deferred@244937534 after retry RPC after error@187520098) -> (continuation of Deferred@202693447 after retry RPC after error@992262245) -> (continuation of Deferred@2120297660 after retry RPC after error@521075030) -> (continuation of Deferred@904582931 after retry RPC after error@731096667) -> (continuation of Deferred@1549826905 after retry RPC after error@885849546), errback=(continuation of Deferred@1924179696 after retry RPC after error@1267320584) -> (continuation of Deferred@1245290720 after retry RPC after error@2045369813) -> (continuation of Deferred@244937534 after retry RPC after error@187520098) -> (continuation of Deferred@202693447 after retry RPC after error@992262245) -> (continuation of Deferred@2120297660 after retry RPC after error@521075030) -> (continuation of Deferred@904582931 after retry RPC after error@731096667) -> (continuation of Deferred@1549826905 after retry RPC after error@885849546)))
21:04:32.624 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:32.624 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:32.625 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:32.625 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:32.625 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:32.625 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:32.625 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:32.625 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:32.626 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:32.644 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:32.644 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:32.645 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:32.645 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:32.645 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:32.645 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:32.645 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:32.645 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:32.646 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:32.664 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:32.664 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:32.665 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:32.665 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:32.665 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:32.665 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:32.665 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:32.665 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:32.666 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:32.684 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:32.684 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:32.685 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:32.685 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:32.685 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:32.685 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:32.685 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:32.685 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:32.686 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:32.704 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:32.705 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:32.705 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:32.705 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:32.705 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:32.705 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:32.705 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:32.705 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:32.706 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:32.724 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:32.724 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:32.725 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:32.725 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:32.725 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:32.725 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:32.725 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:32.725 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:32.726 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:32.764 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:32.765 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:32.765 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:32.765 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:32.765 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:32.765 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:32.765 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:32.765 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:32.766 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:32.767 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=142), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [141ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [141ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [142ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, deferred=Deferred@2111589499(state=PENDING, result=null, callback=(continuation of Deferred@1414929474 after retry RPC after error@2084529455) -> (continuation of Deferred@600656900 after retry RPC after error@1013801874) -> (continuation of Deferred@1488837713 after retry RPC after error@1605164148) -> (continuation of Deferred@373823209 after retry RPC after error@1097766465) -> (continuation of Deferred@2005374313 after retry RPC after error@1324464273) -> (continuation of Deferred@408420600 after retry RPC after error@766595372), errback=(continuation of Deferred@1414929474 after retry RPC after error@2084529455) -> (continuation of Deferred@600656900 after retry RPC after error@1013801874) -> (continuation of Deferred@1488837713 after retry RPC after error@1605164148) -> (continuation of Deferred@373823209 after retry RPC after error@1097766465) -> (continuation of Deferred@2005374313 after retry RPC after error@1324464273) -> (continuation of Deferred@408420600 after retry RPC after error@766595372)))
21:04:32.767 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=142), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [141ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [141ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [142ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, deferred=Deferred@2111589499(state=PENDING, result=null, callback=(continuation of Deferred@1414929474 after retry RPC after error@2084529455) -> (continuation of Deferred@600656900 after retry RPC after error@1013801874) -> (continuation of Deferred@1488837713 after retry RPC after error@1605164148) -> (continuation of Deferred@373823209 after retry RPC after error@1097766465) -> (continuation of Deferred@2005374313 after retry RPC after error@1324464273) -> (continuation of Deferred@408420600 after retry RPC after error@766595372), errback=(continuation of Deferred@1414929474 after retry RPC after error@2084529455) -> (continuation of Deferred@600656900 after retry RPC after error@1013801874) -> (continuation of Deferred@1488837713 after retry RPC after error@1605164148) -> (continuation of Deferred@373823209 after retry RPC after error@1097766465) -> (continuation of Deferred@2005374313 after retry RPC after error@1324464273) -> (continuation of Deferred@408420600 after retry RPC after error@766595372)))
21:04:32.784 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:32.784 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:32.785 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:32.785 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:32.785 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:32.785 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:32.785 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:32.785 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:32.786 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:32.804 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:32.804 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:32.805 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:32.805 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:32.805 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:32.805 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:32.805 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:32.805 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:32.806 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:32.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:32.815289 23569 master_proxy_rpc.cc:177] Failed to determine new leader Master: Timed out: ConnectToClusterRpc(addrs: 127.22.168.62:37239,127.22.168.61:32947,127.22.168.60:33247, num_attempts: 97) passed its deadline: Not found: no leader found: ConnectToClusterRpc(addrs: 127.22.168.62:37239,127.22.168.61:32947,127.22.168.60:33247, num_attempts: 1)
21:04:32.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:32.816740 23569 meta_cache.cc:788] Timed out: LookupRpcById { tablet: '01aa0e192c434b468ec7ed90d48563b3', attempt: 2 } failed: LookupRpcById timed out after deadline expired: LookupRpcById { tablet: '01aa0e192c434b468ec7ed90d48563b3', attempt: 2 } passed its deadline: Timed out: ConnectToClusterRpc(addrs: 127.22.168.62:37239,127.22.168.61:32947,127.22.168.60:33247, num_attempts: 97) passed its deadline: Not found: no leader found: ConnectToClusterRpc(addrs: 127.22.168.62:37239,127.22.168.61:32947,127.22.168.60:33247, num_attempts: 1)
21:04:32.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:32.818821 23569 master_proxy_rpc.cc:203] Re-attempting LookupRpcById request to leader Master (127.22.168.60:33247)
21:04:32.824 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:32.824 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:32.825 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:32.825 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:32.825 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:32.825 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:32.825 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:32.825 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:32.826 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:32.844 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:32.844 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:32.845 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:32.845 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:32.845 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:32.845 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:32.845 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:32.845 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:32.845 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:32.864 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:32.865 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:32.865 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:32.865 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:32.865 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:32.866 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:32.866 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:32.866 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:32.866 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:32.884 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:32.885 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:32.885 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:32.885 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:32.885 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:32.885 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:32.885 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:32.885 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:32.886 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:32.965 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:32.965 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:32.965 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:32.965 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:32.966 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:32.966 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:32.966 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:32.966 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:32.967 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:32.984 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:32.985 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:32.985 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:32.985 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:32.985 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:32.985 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:32.986 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:32.986 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:32.986 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:32.987 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=202), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [59ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [61ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [182ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [182ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [183ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [183ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [201ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [202ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [202ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, deferred=Deferred@31211136(state=PENDING, result=null, callback=(continuation of Deferred@1596176840 after retry RPC after error@1888843529) -> (continuation of Deferred@178902205 after retry RPC after error@1793184598) -> (continuation of Deferred@676256315 after retry RPC after error@1941495044) -> (continuation of Deferred@930145266 after retry RPC after error@1408232745) -> (continuation of Deferred@229538413 after retry RPC after error@944490017) -> (continuation of Deferred@135585193 after retry RPC after error@616496935) -> (continuation of Deferred@1513985231 after retry RPC after error@1008810855), errback=(continuation of Deferred@1596176840 after retry RPC after error@1888843529) -> (continuation of Deferred@178902205 after retry RPC after error@1793184598) -> (continuation of Deferred@676256315 after retry RPC after error@1941495044) -> (continuation of Deferred@930145266 after retry RPC after error@1408232745) -> (continuation of Deferred@229538413 after retry RPC after error@944490017) -> (continuation of Deferred@135585193 after retry RPC after error@616496935) -> (continuation of Deferred@1513985231 after retry RPC after error@1008810855)))
21:04:32.987 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=202), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [59ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [61ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [182ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [182ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [183ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [183ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [201ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [202ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [202ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, deferred=Deferred@31211136(state=PENDING, result=null, callback=(continuation of Deferred@1596176840 after retry RPC after error@1888843529) -> (continuation of Deferred@178902205 after retry RPC after error@1793184598) -> (continuation of Deferred@676256315 after retry RPC after error@1941495044) -> (continuation of Deferred@930145266 after retry RPC after error@1408232745) -> (continuation of Deferred@229538413 after retry RPC after error@944490017) -> (continuation of Deferred@135585193 after retry RPC after error@616496935) -> (continuation of Deferred@1513985231 after retry RPC after error@1008810855), errback=(continuation of Deferred@1596176840 after retry RPC after error@1888843529) -> (continuation of Deferred@178902205 after retry RPC after error@1793184598) -> (continuation of Deferred@676256315 after retry RPC after error@1941495044) -> (continuation of Deferred@930145266 after retry RPC after error@1408232745) -> (continuation of Deferred@229538413 after retry RPC after error@944490017) -> (continuation of Deferred@135585193 after retry RPC after error@616496935) -> (continuation of Deferred@1513985231 after retry RPC after error@1008810855)))
21:04:33.004 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:33.005 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:33.005 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:33.005 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:33.006 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:33.006 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:33.006 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:33.006 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:33.007 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:33.024 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:33.025 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:33.025 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:33.025 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:33.025 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:33.025 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:33.025 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:33.025 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:33.027 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:33.044 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:33.044 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:33.045 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:33.045 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:33.045 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:33.045 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:33.045 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:33.045 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:33.046 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:33.064 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:33.064 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:33.065 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:33.065 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:33.065 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:33.065 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:33.065 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:33.065 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:33.066 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:33.084 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:33.084 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:33.085 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:33.085 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:33.085 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:33.085 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:33.085 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:33.085 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:33.086 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:33.124 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:33.124 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:33.125 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:33.125 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:33.125 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:33.125 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:33.125 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:33.125 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:33.126 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:33.144 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:33.144 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:33.145 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:33.145 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:33.145 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:33.145 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:33.145 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:33.145 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:33.146 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:33.164 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:33.164 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:33.165 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:33.165 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:33.165 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:33.165 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:33.165 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:33.165 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:33.166 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:33.166 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=161), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [3ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [141ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [141ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [142ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [161ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [161ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [162ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, deferred=Deferred@766488176(state=PENDING, result=null, callback=(continuation of Deferred@52871357 after retry RPC after error@2048797633) -> (continuation of Deferred@382787186 after retry RPC after error@1188835062) -> (continuation of Deferred@1942633912 after retry RPC after error@2013347875) -> (continuation of Deferred@1756504607 after retry RPC after error@1329465953) -> (continuation of Deferred@1841023427 after retry RPC after error@533306383) -> (continuation of Deferred@1712514867 after retry RPC after error@1497400466) -> (continuation of Deferred@1778822056 after retry RPC after error@964459267), errback=(continuation of Deferred@52871357 after retry RPC after error@2048797633) -> (continuation of Deferred@382787186 after retry RPC after error@1188835062) -> (continuation of Deferred@1942633912 after retry RPC after error@2013347875) -> (continuation of Deferred@1756504607 after retry RPC after error@1329465953) -> (continuation of Deferred@1841023427 after retry RPC after error@533306383) -> (continuation of Deferred@1712514867 after retry RPC after error@1497400466) -> (continuation of Deferred@1778822056 after retry RPC after error@964459267)))
21:04:33.167 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=161), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [3ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [141ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [141ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [142ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [161ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [161ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [162ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, deferred=Deferred@766488176(state=PENDING, result=null, callback=(continuation of Deferred@52871357 after retry RPC after error@2048797633) -> (continuation of Deferred@382787186 after retry RPC after error@1188835062) -> (continuation of Deferred@1942633912 after retry RPC after error@2013347875) -> (continuation of Deferred@1756504607 after retry RPC after error@1329465953) -> (continuation of Deferred@1841023427 after retry RPC after error@533306383) -> (continuation of Deferred@1712514867 after retry RPC after error@1497400466) -> (continuation of Deferred@1778822056 after retry RPC after error@964459267), errback=(continuation of Deferred@52871357 after retry RPC after error@2048797633) -> (continuation of Deferred@382787186 after retry RPC after error@1188835062) -> (continuation of Deferred@1942633912 after retry RPC after error@2013347875) -> (continuation of Deferred@1756504607 after retry RPC after error@1329465953) -> (continuation of Deferred@1841023427 after retry RPC after error@533306383) -> (continuation of Deferred@1712514867 after retry RPC after error@1497400466) -> (continuation of Deferred@1778822056 after retry RPC after error@964459267)))
21:04:33.184 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:33.184 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:33.185 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:33.185 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:33.185 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:33.185 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:33.185 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:33.185 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:33.186 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:33.204 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:33.204 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:33.205 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:33.205 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:33.205 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:33.205 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:33.205 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:33.205 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:33.206 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:33.224 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:33.224 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:33.225 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:33.225 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:33.225 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:33.225 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:33.225 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:33.225 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:33.226 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:33.244 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:33.244 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:33.245 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:33.245 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:33.245 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:33.246 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:33.246 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:33.246 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:33.246 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:33.264 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:33.264 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:33.265 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:33.265 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:33.265 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:33.265 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:33.265 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:33.265 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:33.266 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:33.284 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:33.284 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:33.285 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:33.285 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:33.285 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:33.285 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:33.285 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:33.285 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:33.286 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:33.304 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:33.304 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:33.305 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:33.305 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:33.305 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:33.305 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:33.305 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:33.305 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:33.306 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:33.364 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:33.364 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:33.365 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:33.365 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:33.365 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:33.365 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:33.365 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:33.365 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:33.366 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:33.367 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=182), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [19ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [62ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [181ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [181ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [182ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, deferred=Deferred@1240061710(state=PENDING, result=null, callback=(continuation of Deferred@1185437107 after retry RPC after error@1120220888) -> (continuation of Deferred@1059663296 after retry RPC after error@956288140) -> (continuation of Deferred@1023280109 after retry RPC after error@1466335580) -> (continuation of Deferred@567800157 after retry RPC after error@1737146951) -> (continuation of Deferred@732386352 after retry RPC after error@1210604665) -> (continuation of Deferred@1458241848 after retry RPC after error@1740794200) -> (continuation of Deferred@1993713476 after retry RPC after error@2090133107), errback=(continuation of Deferred@1185437107 after retry RPC after error@1120220888) -> (continuation of Deferred@1059663296 after retry RPC after error@956288140) -> (continuation of Deferred@1023280109 after retry RPC after error@1466335580) -> (continuation of Deferred@567800157 after retry RPC after error@1737146951) -> (continuation of Deferred@732386352 after retry RPC after error@1210604665) -> (continuation of Deferred@1458241848 after retry RPC after error@1740794200) -> (continuation of Deferred@1993713476 after retry RPC after error@2090133107)))
21:04:33.367 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=182), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [19ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [62ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [181ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [181ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [182ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, deferred=Deferred@1240061710(state=PENDING, result=null, callback=(continuation of Deferred@1185437107 after retry RPC after error@1120220888) -> (continuation of Deferred@1059663296 after retry RPC after error@956288140) -> (continuation of Deferred@1023280109 after retry RPC after error@1466335580) -> (continuation of Deferred@567800157 after retry RPC after error@1737146951) -> (continuation of Deferred@732386352 after retry RPC after error@1210604665) -> (continuation of Deferred@1458241848 after retry RPC after error@1740794200) -> (continuation of Deferred@1993713476 after retry RPC after error@2090133107), errback=(continuation of Deferred@1185437107 after retry RPC after error@1120220888) -> (continuation of Deferred@1059663296 after retry RPC after error@956288140) -> (continuation of Deferred@1023280109 after retry RPC after error@1466335580) -> (continuation of Deferred@567800157 after retry RPC after error@1737146951) -> (continuation of Deferred@732386352 after retry RPC after error@1210604665) -> (continuation of Deferred@1458241848 after retry RPC after error@1740794200) -> (continuation of Deferred@1993713476 after retry RPC after error@2090133107)))
21:04:33.384 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:33.384 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:33.385 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:33.385 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:33.385 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:33.385 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:33.385 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:33.385 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:33.386 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:33.404 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:33.404 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:33.405 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:33.405 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:33.405 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:33.405 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:33.405 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:33.405 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:33.406 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:33.424 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:33.424 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:33.425 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:33.425 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:33.425 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:33.425 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:33.425 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:33.425 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:33.426 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:33.444 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:33.444 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:33.445 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:33.445 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:33.445 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:33.445 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:33.445 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:33.445 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:33.445 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:33.464 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:33.464 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:33.465 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:33.465 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:33.465 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:33.465 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:33.465 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:33.465 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:33.466 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:33.504 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:33.504 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:33.505 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:33.505 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:33.505 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:33.505 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:33.505 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:33.505 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:33.506 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:33.524 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:33.524 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:33.525 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:33.525 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:33.525 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:33.525 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:33.525 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:33.525 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:33.526 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:33.544 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:33.544 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:33.545 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:33.545 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:33.545 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:33.545 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:33.545 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:33.545 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:33.546 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:33.546 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=162), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [79ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [139ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [141ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [141ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [142ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [159ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [161ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [161ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [162ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, deferred=Deferred@557255001(state=PENDING, result=null, callback=(continuation of Deferred@433315836 after retry RPC after error@1915759930) -> (continuation of Deferred@1649028483 after retry RPC after error@2075037010) -> (continuation of Deferred@2080739812 after retry RPC after error@263137955) -> (continuation of Deferred@1010448608 after retry RPC after error@871495208) -> (continuation of Deferred@1672207107 after retry RPC after error@354885607) -> (continuation of Deferred@2025314459 after retry RPC after error@1154162963) -> (continuation of Deferred@2038798666 after retry RPC after error@1416653242), errback=(continuation of Deferred@433315836 after retry RPC after error@1915759930) -> (continuation of Deferred@1649028483 after retry RPC after error@2075037010) -> (continuation of Deferred@2080739812 after retry RPC after error@263137955) -> (continuation of Deferred@1010448608 after retry RPC after error@871495208) -> (continuation of Deferred@1672207107 after retry RPC after error@354885607) -> (continuation of Deferred@2025314459 after retry RPC after error@1154162963) -> (continuation of Deferred@2038798666 after retry RPC after error@1416653242)))
21:04:33.547 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=162), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [79ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [139ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [141ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [141ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [142ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [159ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [161ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [161ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [162ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, deferred=Deferred@557255001(state=PENDING, result=null, callback=(continuation of Deferred@433315836 after retry RPC after error@1915759930) -> (continuation of Deferred@1649028483 after retry RPC after error@2075037010) -> (continuation of Deferred@2080739812 after retry RPC after error@263137955) -> (continuation of Deferred@1010448608 after retry RPC after error@871495208) -> (continuation of Deferred@1672207107 after retry RPC after error@354885607) -> (continuation of Deferred@2025314459 after retry RPC after error@1154162963) -> (continuation of Deferred@2038798666 after retry RPC after error@1416653242), errback=(continuation of Deferred@433315836 after retry RPC after error@1915759930) -> (continuation of Deferred@1649028483 after retry RPC after error@2075037010) -> (continuation of Deferred@2080739812 after retry RPC after error@263137955) -> (continuation of Deferred@1010448608 after retry RPC after error@871495208) -> (continuation of Deferred@1672207107 after retry RPC after error@354885607) -> (continuation of Deferred@2025314459 after retry RPC after error@1154162963) -> (continuation of Deferred@2038798666 after retry RPC after error@1416653242)))
21:04:33.565 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:33.565 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:33.565 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:33.565 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:33.566 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:33.566 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:33.566 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:33.566 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:33.567 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:33.584 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:33.585 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:33.585 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:33.585 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:33.585 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:33.585 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:33.586 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:33.586 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:33.587 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:33.604 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:33.605 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:33.605 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:33.605 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:33.606 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:33.606 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:33.606 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:33.606 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:33.606 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:33.624 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:33.624 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:33.625 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:33.625 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:33.625 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:33.625 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:33.625 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:33.625 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:33.626 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:33.644 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:33.644 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:33.645 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:33.645 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:33.645 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:33.645 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:33.645 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:33.645 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:33.646 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:33.664 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:33.664 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:33.665 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:33.665 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:33.665 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:33.665 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:33.665 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:33.665 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:33.666 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:33.684 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:33.684 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:33.685 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:33.685 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:33.685 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:33.685 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:33.685 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:33.685 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:33.686 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:33.784 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:33.785 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:33.785 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:33.785 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:33.785 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:33.785 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:33.786 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:33.786 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:33.787 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:33.787 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=222), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [3ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [99ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [221ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [222ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [222ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [223ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, deferred=Deferred@1338543780(state=PENDING, result=null, callback=(continuation of Deferred@2139268274 after retry RPC after error@1364211552) -> (continuation of Deferred@1145289781 after retry RPC after error@1333110711) -> (continuation of Deferred@624438751 after retry RPC after error@1963193647) -> (continuation of Deferred@754474414 after retry RPC after error@1322393497) -> (continuation of Deferred@710968649 after retry RPC after error@1075881895) -> (continuation of Deferred@777750304 after retry RPC after error@1881893797) -> (continuation of Deferred@1582297636 after retry RPC after error@462908373), errback=(continuation of Deferred@2139268274 after retry RPC after error@1364211552) -> (continuation of Deferred@1145289781 after retry RPC after error@1333110711) -> (continuation of Deferred@624438751 after retry RPC after error@1963193647) -> (continuation of Deferred@754474414 after retry RPC after error@1322393497) -> (continuation of Deferred@710968649 after retry RPC after error@1075881895) -> (continuation of Deferred@777750304 after retry RPC after error@1881893797) -> (continuation of Deferred@1582297636 after retry RPC after error@462908373)))
21:04:33.788 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=222), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [3ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [99ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [221ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [222ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [222ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [223ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, deferred=Deferred@1338543780(state=PENDING, result=null, callback=(continuation of Deferred@2139268274 after retry RPC after error@1364211552) -> (continuation of Deferred@1145289781 after retry RPC after error@1333110711) -> (continuation of Deferred@624438751 after retry RPC after error@1963193647) -> (continuation of Deferred@754474414 after retry RPC after error@1322393497) -> (continuation of Deferred@710968649 after retry RPC after error@1075881895) -> (continuation of Deferred@777750304 after retry RPC after error@1881893797) -> (continuation of Deferred@1582297636 after retry RPC after error@462908373), errback=(continuation of Deferred@2139268274 after retry RPC after error@1364211552) -> (continuation of Deferred@1145289781 after retry RPC after error@1333110711) -> (continuation of Deferred@624438751 after retry RPC after error@1963193647) -> (continuation of Deferred@754474414 after retry RPC after error@1322393497) -> (continuation of Deferred@710968649 after retry RPC after error@1075881895) -> (continuation of Deferred@777750304 after retry RPC after error@1881893797) -> (continuation of Deferred@1582297636 after retry RPC after error@462908373)))
21:04:33.804 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:33.805 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:33.806 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:33.806 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:33.806 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:33.806 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:33.806 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:33.806 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:33.807 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:33.824 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:33.825 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:33.825 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:33.825 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:33.825 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:33.825 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:33.826 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:33.826 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:33.826 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:33.844 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:33.845 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:33.845 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:33.845 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:33.845 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:33.845 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:33.845 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:33.845 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:33.846 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:33.864 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:33.865 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:33.865 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:33.865 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:33.865 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:33.865 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:33.865 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:33.865 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:33.866 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:33.884 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:33.885 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:33.885 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:33.885 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:33.885 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:33.886 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:33.886 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:33.886 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:33.887 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:33.904 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:33.905 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:33.905 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:33.905 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:33.905 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:33.906 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:33.906 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:33.906 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:33.907 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:33.964 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:33.964 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:33.965 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:33.965 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:33.965 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:33.965 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:33.965 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:33.965 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:33.966 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:34.024 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:34.024 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:34.025 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:34.025 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:34.025 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:34.025 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:34.025 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:34.025 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:34.026 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:34.027 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=222), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [3ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [83ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [103ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [159ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [161ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [161ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [162ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [221ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [221ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [222ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [222ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, deferred=Deferred@1215730038(state=PENDING, result=null, callback=(continuation of Deferred@528431650 after retry RPC after error@1185786634) -> (continuation of Deferred@305470306 after retry RPC after error@1848956275) -> (continuation of Deferred@150756541 after retry RPC after error@597840723) -> (continuation of Deferred@427205198 after retry RPC after error@1361142622) -> (continuation of Deferred@1003668595 after retry RPC after error@173744273) -> (continuation of Deferred@638818118 after retry RPC after error@2087861065) -> (continuation of Deferred@1370397160 after retry RPC after error@35808484), errback=(continuation of Deferred@528431650 after retry RPC after error@1185786634) -> (continuation of Deferred@305470306 after retry RPC after error@1848956275) -> (continuation of Deferred@150756541 after retry RPC after error@597840723) -> (continuation of Deferred@427205198 after retry RPC after error@1361142622) -> (continuation of Deferred@1003668595 after retry RPC after error@173744273) -> (continuation of Deferred@638818118 after retry RPC after error@2087861065) -> (continuation of Deferred@1370397160 after retry RPC after error@35808484)))
21:04:34.027 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=222), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [3ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [83ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [103ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [159ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [161ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [161ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [162ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [221ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [221ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [222ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [222ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, deferred=Deferred@1215730038(state=PENDING, result=null, callback=(continuation of Deferred@528431650 after retry RPC after error@1185786634) -> (continuation of Deferred@305470306 after retry RPC after error@1848956275) -> (continuation of Deferred@150756541 after retry RPC after error@597840723) -> (continuation of Deferred@427205198 after retry RPC after error@1361142622) -> (continuation of Deferred@1003668595 after retry RPC after error@173744273) -> (continuation of Deferred@638818118 after retry RPC after error@2087861065) -> (continuation of Deferred@1370397160 after retry RPC after error@35808484), errback=(continuation of Deferred@528431650 after retry RPC after error@1185786634) -> (continuation of Deferred@305470306 after retry RPC after error@1848956275) -> (continuation of Deferred@150756541 after retry RPC after error@597840723) -> (continuation of Deferred@427205198 after retry RPC after error@1361142622) -> (continuation of Deferred@1003668595 after retry RPC after error@173744273) -> (continuation of Deferred@638818118 after retry RPC after error@2087861065) -> (continuation of Deferred@1370397160 after retry RPC after error@35808484)))
21:04:34.044 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:34.044 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:34.045 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:34.045 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:34.045 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:34.045 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:34.045 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:34.045 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:34.046 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:34.064 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:34.064 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:34.065 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:34.065 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:34.065 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:34.065 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:34.065 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:34.065 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:34.066 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:34.084 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:34.084 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:34.085 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:34.085 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:34.085 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:34.085 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:34.085 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:34.085 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:34.086 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:34.104 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:34.104 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:34.105 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:34.105 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:34.105 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:34.105 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:34.105 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:34.105 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:34.106 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:34.124 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:34.124 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:34.125 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:34.125 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:34.125 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:34.125 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:34.125 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:34.125 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:34.126 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:34.164 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:34.164 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:34.165 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:34.165 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:34.165 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:34.165 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:34.165 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:34.165 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:34.166 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:34.184 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:34.184 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:34.185 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:34.185 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:34.185 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:34.185 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:34.185 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:34.185 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:34.186 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:34.224 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:34.225 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:34.225 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:34.225 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:34.225 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:34.225 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:34.225 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:34.225 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:34.227 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:34.227 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=183), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [39ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [59ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [79ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [141ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [141ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [142ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [181ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [181ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [183ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [183ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, deferred=Deferred@1990684921(state=PENDING, result=null, callback=(continuation of Deferred@622916178 after retry RPC after error@743758019) -> (continuation of Deferred@375132850 after retry RPC after error@1529976888) -> (continuation of Deferred@2040615790 after retry RPC after error@1933123161) -> (continuation of Deferred@92687118 after retry RPC after error@1377956830) -> (continuation of Deferred@568914135 after retry RPC after error@437840229) -> (continuation of Deferred@762384147 after retry RPC after error@1880335968) -> (continuation of Deferred@370904829 after retry RPC after error@1268063898), errback=(continuation of Deferred@622916178 after retry RPC after error@743758019) -> (continuation of Deferred@375132850 after retry RPC after error@1529976888) -> (continuation of Deferred@2040615790 after retry RPC after error@1933123161) -> (continuation of Deferred@92687118 after retry RPC after error@1377956830) -> (continuation of Deferred@568914135 after retry RPC after error@437840229) -> (continuation of Deferred@762384147 after retry RPC after error@1880335968) -> (continuation of Deferred@370904829 after retry RPC after error@1268063898)))
21:04:34.228 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=183), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [39ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [59ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [79ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [141ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [141ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [142ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [181ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [181ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [183ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [183ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, deferred=Deferred@1990684921(state=PENDING, result=null, callback=(continuation of Deferred@622916178 after retry RPC after error@743758019) -> (continuation of Deferred@375132850 after retry RPC after error@1529976888) -> (continuation of Deferred@2040615790 after retry RPC after error@1933123161) -> (continuation of Deferred@92687118 after retry RPC after error@1377956830) -> (continuation of Deferred@568914135 after retry RPC after error@437840229) -> (continuation of Deferred@762384147 after retry RPC after error@1880335968) -> (continuation of Deferred@370904829 after retry RPC after error@1268063898), errback=(continuation of Deferred@622916178 after retry RPC after error@743758019) -> (continuation of Deferred@375132850 after retry RPC after error@1529976888) -> (continuation of Deferred@2040615790 after retry RPC after error@1933123161) -> (continuation of Deferred@92687118 after retry RPC after error@1377956830) -> (continuation of Deferred@568914135 after retry RPC after error@437840229) -> (continuation of Deferred@762384147 after retry RPC after error@1880335968) -> (continuation of Deferred@370904829 after retry RPC after error@1268063898)))
21:04:34.244 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:34.244 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:34.245 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:34.245 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:34.245 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:34.245 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:34.245 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:34.246 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:34.246 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:34.264 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:34.264 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:34.265 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:34.265 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:34.265 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:34.265 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:34.265 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:34.265 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:34.266 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:34.284 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:34.284 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:34.285 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:34.285 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:34.285 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:34.285 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:34.285 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:34.285 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:34.286 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:34.304 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:34.304 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:34.305 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:34.305 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:34.305 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:34.305 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:34.305 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:34.305 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:34.306 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:34.324 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:34.324 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:34.325 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:34.325 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:34.325 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:34.325 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:34.325 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:34.325 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:34.326 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:34.344 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:34.344 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:34.345 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:34.345 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:34.345 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:34.345 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:34.345 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:34.345 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:34.346 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:34.404 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:34.405 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:34.405 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:34.405 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:34.405 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:34.405 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:34.405 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:34.406 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:34.406 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:34.406 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:34.407 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:34.407 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:34.444 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:34.445 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:34.445 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:34.445 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:34.445 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:34.445 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:34.446 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:34.446 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:34.447 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:34.448 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=203), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [161ms] refreshing cache from master, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [162ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [162ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [162ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [163ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [202ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [202ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [203ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [203ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, deferred=Deferred@1210345821(state=PENDING, result=null, callback=(continuation of Deferred@1512815983 after retry RPC after error@107510888) -> (continuation of Deferred@1020770196 after retry RPC after error@1465012178) -> (continuation of Deferred@1343570066 after retry RPC after error@358346180) -> (continuation of Deferred@338793452 after retry RPC after error@1373135463) -> (continuation of Deferred@733198622 after retry RPC after error@1632392289) -> (continuation of Deferred@2038039895 after retry RPC after error@1805880133) -> (continuation of Deferred@1575920742 after retry RPC after error@1577192678), errback=(continuation of Deferred@1512815983 after retry RPC after error@107510888) -> (continuation of Deferred@1020770196 after retry RPC after error@1465012178) -> (continuation of Deferred@1343570066 after retry RPC after error@358346180) -> (continuation of Deferred@338793452 after retry RPC after error@1373135463) -> (continuation of Deferred@733198622 after retry RPC after error@1632392289) -> (continuation of Deferred@2038039895 after retry RPC after error@1805880133) -> (continuation of Deferred@1575920742 after retry RPC after error@1577192678)))
21:04:34.448 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=203), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [161ms] refreshing cache from master, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [162ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [162ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [162ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [163ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [202ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [202ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [203ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [203ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, deferred=Deferred@1210345821(state=PENDING, result=null, callback=(continuation of Deferred@1512815983 after retry RPC after error@107510888) -> (continuation of Deferred@1020770196 after retry RPC after error@1465012178) -> (continuation of Deferred@1343570066 after retry RPC after error@358346180) -> (continuation of Deferred@338793452 after retry RPC after error@1373135463) -> (continuation of Deferred@733198622 after retry RPC after error@1632392289) -> (continuation of Deferred@2038039895 after retry RPC after error@1805880133) -> (continuation of Deferred@1575920742 after retry RPC after error@1577192678), errback=(continuation of Deferred@1512815983 after retry RPC after error@107510888) -> (continuation of Deferred@1020770196 after retry RPC after error@1465012178) -> (continuation of Deferred@1343570066 after retry RPC after error@358346180) -> (continuation of Deferred@338793452 after retry RPC after error@1373135463) -> (continuation of Deferred@733198622 after retry RPC after error@1632392289) -> (continuation of Deferred@2038039895 after retry RPC after error@1805880133) -> (continuation of Deferred@1575920742 after retry RPC after error@1577192678)))
21:04:34.464 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:34.465 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:34.465 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:34.465 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:34.465 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:34.465 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:34.466 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:34.466 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:34.466 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:34.484 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:34.485 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:34.485 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:34.485 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:34.485 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:34.485 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:34.486 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:34.486 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:34.487 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:34.504 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:34.505 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:34.505 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:34.505 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:34.505 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:34.505 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:34.505 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:34.505 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:34.506 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:34.524 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:34.524 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:34.525 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:34.525 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:34.525 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:34.525 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:34.525 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:34.525 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:34.526 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:34.544 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:34.544 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:34.545 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:34.545 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:34.545 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:34.545 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:34.545 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:34.545 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:34.546 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:34.564 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:34.564 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:34.565 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:34.565 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:34.565 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:34.565 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:34.565 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:34.565 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:34.566 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:34.624 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:34.625 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:34.625 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:34.625 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:34.625 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:34.625 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:34.626 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:34.626 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:34.627 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:34.627 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=162), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [162ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [162ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [162ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [163ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, deferred=Deferred@2019650012(state=PENDING, result=null, callback=(continuation of Deferred@553874500 after retry RPC after error@861198006) -> (continuation of Deferred@972320008 after retry RPC after error@1244191843) -> (continuation of Deferred@1882398774 after retry RPC after error@1699199456) -> (continuation of Deferred@2143356013 after retry RPC after error@1929774130) -> (continuation of Deferred@442345716 after retry RPC after error@1116576486) -> (continuation of Deferred@890772662 after retry RPC after error@1987881395), errback=(continuation of Deferred@553874500 after retry RPC after error@861198006) -> (continuation of Deferred@972320008 after retry RPC after error@1244191843) -> (continuation of Deferred@1882398774 after retry RPC after error@1699199456) -> (continuation of Deferred@2143356013 after retry RPC after error@1929774130) -> (continuation of Deferred@442345716 after retry RPC after error@1116576486) -> (continuation of Deferred@890772662 after retry RPC after error@1987881395)))
21:04:34.627 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=162), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [162ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [162ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [162ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [163ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, deferred=Deferred@2019650012(state=PENDING, result=null, callback=(continuation of Deferred@553874500 after retry RPC after error@861198006) -> (continuation of Deferred@972320008 after retry RPC after error@1244191843) -> (continuation of Deferred@1882398774 after retry RPC after error@1699199456) -> (continuation of Deferred@2143356013 after retry RPC after error@1929774130) -> (continuation of Deferred@442345716 after retry RPC after error@1116576486) -> (continuation of Deferred@890772662 after retry RPC after error@1987881395), errback=(continuation of Deferred@553874500 after retry RPC after error@861198006) -> (continuation of Deferred@972320008 after retry RPC after error@1244191843) -> (continuation of Deferred@1882398774 after retry RPC after error@1699199456) -> (continuation of Deferred@2143356013 after retry RPC after error@1929774130) -> (continuation of Deferred@442345716 after retry RPC after error@1116576486) -> (continuation of Deferred@890772662 after retry RPC after error@1987881395)))
21:04:34.644 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:34.645 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:34.645 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:34.645 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:34.645 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:34.645 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:34.645 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:34.646 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:34.646 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:34.664 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:34.665 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:34.665 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:34.665 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:34.665 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:34.665 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:34.665 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:34.666 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:34.666 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:34.684 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:34.685 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:34.685 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:34.685 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:34.685 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:34.685 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:34.686 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:34.686 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:34.687 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:34.704 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:34.705 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:34.705 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:34.705 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:34.705 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:34.705 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:34.705 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:34.706 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:34.707 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:34.724 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:34.725 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:34.725 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:34.725 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:34.725 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:34.725 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:34.726 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:34.726 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:34.726 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:34.764 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:34.765 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:34.765 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:34.765 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:34.765 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:34.765 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:34.766 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:34.766 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:34.766 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:34.804 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:34.805 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:34.805 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:34.805 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:34.805 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:34.805 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:34.805 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:34.806 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:34.806 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:34.807 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=162), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [161ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [161ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [162ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, deferred=Deferred@1318903288(state=PENDING, result=null, callback=(continuation of Deferred@598610682 after retry RPC after error@448808574) -> (continuation of Deferred@1493185418 after retry RPC after error@1901807183) -> (continuation of Deferred@582067036 after retry RPC after error@979034818) -> (continuation of Deferred@1646140000 after retry RPC after error@1230786898) -> (continuation of Deferred@1717084262 after retry RPC after error@200872605) -> (continuation of Deferred@421975192 after retry RPC after error@853266479), errback=(continuation of Deferred@598610682 after retry RPC after error@448808574) -> (continuation of Deferred@1493185418 after retry RPC after error@1901807183) -> (continuation of Deferred@582067036 after retry RPC after error@979034818) -> (continuation of Deferred@1646140000 after retry RPC after error@1230786898) -> (continuation of Deferred@1717084262 after retry RPC after error@200872605) -> (continuation of Deferred@421975192 after retry RPC after error@853266479)))
21:04:34.807 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=162), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [161ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [161ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [162ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, deferred=Deferred@1318903288(state=PENDING, result=null, callback=(continuation of Deferred@598610682 after retry RPC after error@448808574) -> (continuation of Deferred@1493185418 after retry RPC after error@1901807183) -> (continuation of Deferred@582067036 after retry RPC after error@979034818) -> (continuation of Deferred@1646140000 after retry RPC after error@1230786898) -> (continuation of Deferred@1717084262 after retry RPC after error@200872605) -> (continuation of Deferred@421975192 after retry RPC after error@853266479), errback=(continuation of Deferred@598610682 after retry RPC after error@448808574) -> (continuation of Deferred@1493185418 after retry RPC after error@1901807183) -> (continuation of Deferred@582067036 after retry RPC after error@979034818) -> (continuation of Deferred@1646140000 after retry RPC after error@1230786898) -> (continuation of Deferred@1717084262 after retry RPC after error@200872605) -> (continuation of Deferred@421975192 after retry RPC after error@853266479)))
21:04:34.824 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:34.825 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:34.825 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:34.825 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:34.826 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:34.826 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:34.826 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:34.826 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:34.827 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:34.844 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:34.845 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:34.845 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:34.845 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:34.845 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:34.846 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:34.846 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:34.846 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:34.846 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:34.864 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:34.865 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:34.865 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:34.865 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:34.865 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:34.865 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:34.865 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:34.865 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:34.866 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:34.884 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:34.885 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:34.885 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:34.885 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:34.885 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:34.885 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:34.885 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:34.885 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:34.886 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:34.904 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:34.904 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:34.905 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:34.905 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:34.905 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:34.905 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:34.905 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:34.905 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:34.906 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:34.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:34.917131 24071 master_runner.cc:186] Error getting master registration for 127.22.168.62:37239: Network error: Client connection negotiation failed: client connection to 127.22.168.62:37239: connect: Connection refused (error 111),
21:04:34.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:34.917333 24071 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
21:04:34.924 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:34.924 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:34.925 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:34.925 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:34.925 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:34.925 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:34.925 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:34.925 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:34.926 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:34.964 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:34.965 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:34.965 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:34.965 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:34.965 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:34.965 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:34.965 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:34.966 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:34.966 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:35.064 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:35.064 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:35.065 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:35.065 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:35.065 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:35.065 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:35.065 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:35.065 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:35.066 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:35.066 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=241), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [141ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [141ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [142ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [241ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [241ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [242ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [242ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, deferred=Deferred@1341907326(state=PENDING, result=null, callback=(continuation of Deferred@2129318461 after retry RPC after error@1140719909) -> (continuation of Deferred@1108768447 after retry RPC after error@1383505810) -> (continuation of Deferred@2049295579 after retry RPC after error@1239483795) -> (continuation of Deferred@951267742 after retry RPC after error@1583330069) -> (continuation of Deferred@176727833 after retry RPC after error@1267000902) -> (continuation of Deferred@1780093289 after retry RPC after error@1233047588) -> (continuation of Deferred@100730493 after retry RPC after error@1792767533), errback=(continuation of Deferred@2129318461 after retry RPC after error@1140719909) -> (continuation of Deferred@1108768447 after retry RPC after error@1383505810) -> (continuation of Deferred@2049295579 after retry RPC after error@1239483795) -> (continuation of Deferred@951267742 after retry RPC after error@1583330069) -> (continuation of Deferred@176727833 after retry RPC after error@1267000902) -> (continuation of Deferred@1780093289 after retry RPC after error@1233047588) -> (continuation of Deferred@100730493 after retry RPC after error@1792767533)))
21:04:35.067 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=241), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [141ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [141ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [142ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [241ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [241ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [242ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [242ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, deferred=Deferred@1341907326(state=PENDING, result=null, callback=(continuation of Deferred@2129318461 after retry RPC after error@1140719909) -> (continuation of Deferred@1108768447 after retry RPC after error@1383505810) -> (continuation of Deferred@2049295579 after retry RPC after error@1239483795) -> (continuation of Deferred@951267742 after retry RPC after error@1583330069) -> (continuation of Deferred@176727833 after retry RPC after error@1267000902) -> (continuation of Deferred@1780093289 after retry RPC after error@1233047588) -> (continuation of Deferred@100730493 after retry RPC after error@1792767533), errback=(continuation of Deferred@2129318461 after retry RPC after error@1140719909) -> (continuation of Deferred@1108768447 after retry RPC after error@1383505810) -> (continuation of Deferred@2049295579 after retry RPC after error@1239483795) -> (continuation of Deferred@951267742 after retry RPC after error@1583330069) -> (continuation of Deferred@176727833 after retry RPC after error@1267000902) -> (continuation of Deferred@1780093289 after retry RPC after error@1233047588) -> (continuation of Deferred@100730493 after retry RPC after error@1792767533)))
21:04:35.084 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:35.084 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:35.085 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:35.085 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:35.085 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:35.085 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:35.085 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:35.085 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:35.085 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:35.104 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:35.105 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:35.105 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:35.105 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:35.106 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:35.106 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:35.106 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:35.106 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:35.106 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:35.124 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:35.125 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:35.125 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:35.125 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:35.125 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:35.125 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:35.125 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:35.126 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:35.126 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:35.144 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:35.145 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:35.145 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:35.145 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:35.145 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:35.145 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:35.145 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:35.145 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:35.146 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:35.164 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:35.164 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:35.165 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:35.165 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:35.165 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:35.165 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:35.165 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:35.165 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:35.166 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:35.204 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:35.204 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:35.205 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:35.205 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:35.205 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:35.205 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:35.205 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:35.205 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:35.206 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:35.224 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:35.224 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:35.225 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:35.225 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:35.225 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:35.225 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:35.225 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:35.225 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:35.226 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:35.345 [DEBUG - pool-40-thread-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=260), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [21ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [22ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [22ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [23ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [23ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [23ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [41ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [42ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [42ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [43ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [43ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [61ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [63ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [81ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [83ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [121ms] refreshing cache from master, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [123ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [141ms] refreshing cache from master, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [142ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [142ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [143ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [143ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, deferred=Deferred@2005477092(state=PENDING, result=null, callback=(continuation of Deferred@149645647 after retry RPC after error@1379249583) -> (continuation of Deferred@1341472812 after retry RPC after error@1903554864) -> (continuation of Deferred@570904384 after retry RPC after error@1562433146) -> (continuation of Deferred@942741970 after retry RPC after error@1712558997) -> (continuation of Deferred@2130099544 after retry RPC after error@633219066) -> (continuation of Deferred@368133398 after retry RPC after error@1862595334) -> (continuation of Deferred@617848771 after retry RPC after error@1879506260), errback=(continuation of Deferred@149645647 after retry RPC after error@1379249583) -> (continuation of Deferred@1341472812 after retry RPC after error@1903554864) -> (continuation of Deferred@570904384 after retry RPC after error@1562433146) -> (continuation of Deferred@942741970 after retry RPC after error@1712558997) -> (continuation of Deferred@2130099544 after retry RPC after error@633219066) -> (continuation of Deferred@368133398 after retry RPC after error@1862595334) -> (continuation of Deferred@617848771 after retry RPC after error@1879506260)))
21:04:35.345 [WARN - pool-40-thread-1] (HashedWheelTimer.java:716) An exception was thrown by TimerTask.
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343) ~[main/:?]
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at java.lang.Thread.run(Thread.java:748) [?:1.8.0_275]
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86)
at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106)
at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198)
at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158)
at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953)
at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923)
at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464)
at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713)
at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701)
at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788)
at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501)
at java.lang.Thread.run(Thread.java:748)
21:04:35.364 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:35.364 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:35.365 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:35.365 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:35.365 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:35.365 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:35.365 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:35.365 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:35.366 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:35.384 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:35.384 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:35.385 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:35.385 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:35.385 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:35.385 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:35.385 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:35.385 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:35.386 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:35.404 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:35.404 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:35.405 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:35.405 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:35.405 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:35.405 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:35.405 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:35.405 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:35.406 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:35.424 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:35.424 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:35.425 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:35.425 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:35.425 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:35.425 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:35.425 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:35.425 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:35.426 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:35.444 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:35.444 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:35.445 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:35.445 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:35.445 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:35.445 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:35.445 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:35.445 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:35.446 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:35.464 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:35.464 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:35.465 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:35.465 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:35.465 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:35.465 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:35.465 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:35.465 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:35.466 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:35.484 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:35.484 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:35.485 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:35.485 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:35.485 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:35.485 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:35.485 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:35.485 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:35.486 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:35.584 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:35.585 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:35.585 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:35.585 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:35.585 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:35.585 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:35.585 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:35.585 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:35.586 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:35.593 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=222), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [221ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [221ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [222ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [222ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, deferred=Deferred@66008177(state=PENDING, result=null, callback=(continuation of Deferred@1187044477 after retry RPC after error@1758918679) -> (continuation of Deferred@905711178 after retry RPC after error@1330313272) -> (continuation of Deferred@254408344 after retry RPC after error@1348514090) -> (continuation of Deferred@87784242 after retry RPC after error@489212053) -> (continuation of Deferred@1567977630 after retry RPC after error@139031547) -> (continuation of Deferred@1450535723 after retry RPC after error@230845712) -> (continuation of Deferred@2076511670 after retry RPC after error@953188254), errback=(continuation of Deferred@1187044477 after retry RPC after error@1758918679) -> (continuation of Deferred@905711178 after retry RPC after error@1330313272) -> (continuation of Deferred@254408344 after retry RPC after error@1348514090) -> (continuation of Deferred@87784242 after retry RPC after error@489212053) -> (continuation of Deferred@1567977630 after retry RPC after error@139031547) -> (continuation of Deferred@1450535723 after retry RPC after error@230845712) -> (continuation of Deferred@2076511670 after retry RPC after error@953188254)))
21:04:35.593 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=222), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [221ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [221ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [222ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [222ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, deferred=Deferred@66008177(state=PENDING, result=null, callback=(continuation of Deferred@1187044477 after retry RPC after error@1758918679) -> (continuation of Deferred@905711178 after retry RPC after error@1330313272) -> (continuation of Deferred@254408344 after retry RPC after error@1348514090) -> (continuation of Deferred@87784242 after retry RPC after error@489212053) -> (continuation of Deferred@1567977630 after retry RPC after error@139031547) -> (continuation of Deferred@1450535723 after retry RPC after error@230845712) -> (continuation of Deferred@2076511670 after retry RPC after error@953188254), errback=(continuation of Deferred@1187044477 after retry RPC after error@1758918679) -> (continuation of Deferred@905711178 after retry RPC after error@1330313272) -> (continuation of Deferred@254408344 after retry RPC after error@1348514090) -> (continuation of Deferred@87784242 after retry RPC after error@489212053) -> (continuation of Deferred@1567977630 after retry RPC after error@139031547) -> (continuation of Deferred@1450535723 after retry RPC after error@230845712) -> (continuation of Deferred@2076511670 after retry RPC after error@953188254)))
21:04:35.604 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:35.605 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:35.605 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:35.605 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:35.605 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:35.605 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:35.605 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:35.605 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:35.606 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:35.624 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:35.625 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:35.625 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:35.625 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:35.625 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:35.625 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:35.625 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:35.625 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:35.626 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:35.644 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:35.644 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:35.645 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:35.645 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:35.645 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:35.645 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:35.645 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:35.645 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:35.646 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:35.664 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:35.664 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:35.665 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:35.665 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:35.665 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:35.665 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:35.665 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:35.665 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:35.666 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:35.684 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:35.684 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:35.685 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:35.685 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:35.685 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:35.685 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:35.685 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:35.685 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:35.686 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:35.704 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:35.704 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:35.705 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:35.705 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:35.705 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:35.705 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:35.705 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:35.705 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:35.706 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:35.764 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:35.764 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:35.765 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:35.765 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:35.765 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:35.765 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:35.765 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:35.765 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:35.766 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:35.766 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=161), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [99ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [161ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [161ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [162ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, deferred=Deferred@1954267964(state=PENDING, result=null, callback=(continuation of Deferred@1323727592 after retry RPC after error@150242705) -> (continuation of Deferred@1308218622 after retry RPC after error@1731300446) -> (continuation of Deferred@444435306 after retry RPC after error@914972089) -> (continuation of Deferred@903094706 after retry RPC after error@1584138282) -> (continuation of Deferred@683470422 after retry RPC after error@1420040813) -> (continuation of Deferred@1089042682 after retry RPC after error@2121930133), errback=(continuation of Deferred@1323727592 after retry RPC after error@150242705) -> (continuation of Deferred@1308218622 after retry RPC after error@1731300446) -> (continuation of Deferred@444435306 after retry RPC after error@914972089) -> (continuation of Deferred@903094706 after retry RPC after error@1584138282) -> (continuation of Deferred@683470422 after retry RPC after error@1420040813) -> (continuation of Deferred@1089042682 after retry RPC after error@2121930133)))
21:04:35.767 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=161), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [99ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [161ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [161ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [162ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, deferred=Deferred@1954267964(state=PENDING, result=null, callback=(continuation of Deferred@1323727592 after retry RPC after error@150242705) -> (continuation of Deferred@1308218622 after retry RPC after error@1731300446) -> (continuation of Deferred@444435306 after retry RPC after error@914972089) -> (continuation of Deferred@903094706 after retry RPC after error@1584138282) -> (continuation of Deferred@683470422 after retry RPC after error@1420040813) -> (continuation of Deferred@1089042682 after retry RPC after error@2121930133), errback=(continuation of Deferred@1323727592 after retry RPC after error@150242705) -> (continuation of Deferred@1308218622 after retry RPC after error@1731300446) -> (continuation of Deferred@444435306 after retry RPC after error@914972089) -> (continuation of Deferred@903094706 after retry RPC after error@1584138282) -> (continuation of Deferred@683470422 after retry RPC after error@1420040813) -> (continuation of Deferred@1089042682 after retry RPC after error@2121930133)))
21:04:35.784 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:35.784 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:35.785 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:35.785 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:35.785 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:35.785 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:35.785 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:35.785 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:35.786 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:35.805 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:35.805 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:35.805 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:35.805 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:35.805 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:35.806 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:35.806 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:35.806 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:35.807 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:35.824 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:35.825 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:35.825 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:35.825 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:35.825 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:35.825 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:35.825 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:35.826 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:35.826 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:35.844 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:35.845 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:35.845 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:35.845 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:35.845 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:35.845 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:35.845 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:35.845 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:35.846 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:35.864 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:35.865 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:35.865 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:35.865 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:35.865 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:35.865 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:35.865 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:35.865 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:35.866 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:35.884 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:35.884 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:35.885 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:35.885 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:35.885 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:35.885 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:35.885 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:35.885 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:35.886 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:35.904 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:35.904 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:35.905 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:35.905 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:35.905 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:35.905 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:35.905 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:35.906 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:35.907 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:35.964 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:35.965 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:35.965 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:35.965 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:35.965 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:35.965 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:35.966 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:35.966 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:35.966 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:36.044 [DEBUG - pool-40-thread-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=260), Traces: [0ms] refreshing cache from master, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [3ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [21ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [22ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [22ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [23ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [23ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [24ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [24ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [41ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [42ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [42ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [43ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [61ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [62ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [63ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [81ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [83ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [101ms] refreshing cache from master, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [103ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [121ms] refreshing cache from master, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [123ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [124ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [124ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [181ms] refreshing cache from master, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [182ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [182ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [182ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [183ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [183ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [184ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, deferred=Deferred@1341195440(state=PENDING, result=null, callback=(continuation of Deferred@1745318822 after retry RPC after error@17009626) -> (continuation of Deferred@1244308314 after retry RPC after error@230287915) -> (continuation of Deferred@1592795769 after retry RPC after error@1127231270) -> (continuation of Deferred@1553953897 after retry RPC after error@506799459) -> (continuation of Deferred@1091120950 after retry RPC after error@50487598) -> (continuation of Deferred@614035739 after retry RPC after error@321935062) -> (continuation of Deferred@466135833 after retry RPC after error@190034527) -> (continuation of Deferred@1689066375 after retry RPC after error@1966765858), errback=(continuation of Deferred@1745318822 after retry RPC after error@17009626) -> (continuation of Deferred@1244308314 after retry RPC after error@230287915) -> (continuation of Deferred@1592795769 after retry RPC after error@1127231270) -> (continuation of Deferred@1553953897 after retry RPC after error@506799459) -> (continuation of Deferred@1091120950 after retry RPC after error@50487598) -> (continuation of Deferred@614035739 after retry RPC after error@321935062) -> (continuation of Deferred@466135833 after retry RPC after error@190034527) -> (continuation of Deferred@1689066375 after retry RPC after error@1966765858)))
21:04:36.045 [WARN - pool-40-thread-1] (HashedWheelTimer.java:716) An exception was thrown by TimerTask.
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343) ~[main/:?]
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at java.lang.Thread.run(Thread.java:748) [?:1.8.0_275]
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86)
at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106)
at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198)
at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158)
at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953)
at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923)
at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464)
at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713)
at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701)
at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788)
at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501)
at java.lang.Thread.run(Thread.java:748)
21:04:36.064 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:36.065 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:36.065 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:36.065 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:36.066 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:36.066 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:36.066 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:36.066 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:36.066 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:36.084 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:36.085 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:36.085 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:36.085 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:36.085 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:36.085 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:36.085 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:36.085 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:36.086 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:36.104 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:36.104 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:36.105 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:36.105 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:36.105 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:36.105 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:36.105 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:36.105 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:36.106 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:36.124 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:36.124 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:36.125 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:36.125 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:36.125 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:36.125 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:36.125 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:36.125 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:36.126 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:36.145 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:36.145 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:36.146 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:36.146 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:36.146 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:36.146 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:36.146 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:36.146 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:36.147 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:36.165 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:36.165 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:36.166 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:36.166 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:36.166 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:36.166 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:36.166 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:36.166 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:36.167 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:36.204 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:36.204 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:36.205 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:36.205 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:36.205 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:36.205 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:36.205 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:36.205 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:36.206 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:36.284 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:36.285 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:36.285 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:36.285 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:36.285 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:36.285 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:36.286 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:36.286 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:36.286 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:36.287 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=222), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [83ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [100ms] refreshing cache from master, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [103ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [141ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [141ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [142ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [221ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [222ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [222ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [222ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, deferred=Deferred@1232443972(state=PENDING, result=null, callback=(continuation of Deferred@45250577 after retry RPC after error@1448311211) -> (continuation of Deferred@1400784565 after retry RPC after error@909922596) -> (continuation of Deferred@668976386 after retry RPC after error@1779203699) -> (continuation of Deferred@1273419369 after retry RPC after error@520555328) -> (continuation of Deferred@1177107063 after retry RPC after error@1340025639) -> (continuation of Deferred@846719275 after retry RPC after error@361605422) -> (continuation of Deferred@444126058 after retry RPC after error@742933777), errback=(continuation of Deferred@45250577 after retry RPC after error@1448311211) -> (continuation of Deferred@1400784565 after retry RPC after error@909922596) -> (continuation of Deferred@668976386 after retry RPC after error@1779203699) -> (continuation of Deferred@1273419369 after retry RPC after error@520555328) -> (continuation of Deferred@1177107063 after retry RPC after error@1340025639) -> (continuation of Deferred@846719275 after retry RPC after error@361605422) -> (continuation of Deferred@444126058 after retry RPC after error@742933777)))
21:04:36.287 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=222), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [83ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [100ms] refreshing cache from master, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [103ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [141ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [141ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [142ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [221ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [222ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [222ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [222ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, deferred=Deferred@1232443972(state=PENDING, result=null, callback=(continuation of Deferred@45250577 after retry RPC after error@1448311211) -> (continuation of Deferred@1400784565 after retry RPC after error@909922596) -> (continuation of Deferred@668976386 after retry RPC after error@1779203699) -> (continuation of Deferred@1273419369 after retry RPC after error@520555328) -> (continuation of Deferred@1177107063 after retry RPC after error@1340025639) -> (continuation of Deferred@846719275 after retry RPC after error@361605422) -> (continuation of Deferred@444126058 after retry RPC after error@742933777), errback=(continuation of Deferred@45250577 after retry RPC after error@1448311211) -> (continuation of Deferred@1400784565 after retry RPC after error@909922596) -> (continuation of Deferred@668976386 after retry RPC after error@1779203699) -> (continuation of Deferred@1273419369 after retry RPC after error@520555328) -> (continuation of Deferred@1177107063 after retry RPC after error@1340025639) -> (continuation of Deferred@846719275 after retry RPC after error@361605422) -> (continuation of Deferred@444126058 after retry RPC after error@742933777)))
21:04:36.304 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:36.304 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:36.305 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:36.305 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:36.305 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:36.305 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:36.306 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:36.306 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:36.306 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:36.324 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:36.324 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:36.325 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:36.325 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:36.325 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:36.325 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:36.325 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:36.325 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:36.326 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:36.344 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:36.344 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:36.345 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:36.345 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:36.345 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:36.345 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:36.345 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:36.345 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:36.346 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:36.364 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:36.364 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:36.365 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:36.365 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:36.365 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:36.365 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:36.365 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:36.365 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:36.366 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:36.385 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:36.385 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:36.385 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:36.385 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:36.386 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:36.386 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:36.386 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:36.386 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:36.387 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:36.425 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:36.425 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:36.425 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:36.425 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:36.426 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:36.426 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:36.426 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:36.426 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:36.426 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:36.444 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:36.445 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:36.445 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:36.445 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:36.445 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:36.445 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:36.445 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:36.446 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:36.446 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:36.525 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:36.525 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:36.526 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:36.526 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:36.526 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:36.526 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:36.526 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:36.526 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:36.527 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:36.527 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=222), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [59ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [141ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [142ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [142ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [222ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [222ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [222ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [223ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, deferred=Deferred@647540690(state=PENDING, result=null, callback=(continuation of Deferred@1845557099 after retry RPC after error@729134291) -> (continuation of Deferred@1498879809 after retry RPC after error@978507142) -> (continuation of Deferred@1392559624 after retry RPC after error@1440031639) -> (continuation of Deferred@878300019 after retry RPC after error@403424923) -> (continuation of Deferred@1307472660 after retry RPC after error@828061861) -> (continuation of Deferred@1260617646 after retry RPC after error@224020303) -> (continuation of Deferred@865268515 after retry RPC after error@1533311972), errback=(continuation of Deferred@1845557099 after retry RPC after error@729134291) -> (continuation of Deferred@1498879809 after retry RPC after error@978507142) -> (continuation of Deferred@1392559624 after retry RPC after error@1440031639) -> (continuation of Deferred@878300019 after retry RPC after error@403424923) -> (continuation of Deferred@1307472660 after retry RPC after error@828061861) -> (continuation of Deferred@1260617646 after retry RPC after error@224020303) -> (continuation of Deferred@865268515 after retry RPC after error@1533311972)))
21:04:36.527 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=222), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [59ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [141ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [142ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [142ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [222ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [222ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [222ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [223ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, deferred=Deferred@647540690(state=PENDING, result=null, callback=(continuation of Deferred@1845557099 after retry RPC after error@729134291) -> (continuation of Deferred@1498879809 after retry RPC after error@978507142) -> (continuation of Deferred@1392559624 after retry RPC after error@1440031639) -> (continuation of Deferred@878300019 after retry RPC after error@403424923) -> (continuation of Deferred@1307472660 after retry RPC after error@828061861) -> (continuation of Deferred@1260617646 after retry RPC after error@224020303) -> (continuation of Deferred@865268515 after retry RPC after error@1533311972), errback=(continuation of Deferred@1845557099 after retry RPC after error@729134291) -> (continuation of Deferred@1498879809 after retry RPC after error@978507142) -> (continuation of Deferred@1392559624 after retry RPC after error@1440031639) -> (continuation of Deferred@878300019 after retry RPC after error@403424923) -> (continuation of Deferred@1307472660 after retry RPC after error@828061861) -> (continuation of Deferred@1260617646 after retry RPC after error@224020303) -> (continuation of Deferred@865268515 after retry RPC after error@1533311972)))
21:04:36.544 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:36.545 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:36.545 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:36.545 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:36.546 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:36.546 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:36.546 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:36.546 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:36.547 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:36.564 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:36.565 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:36.565 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:36.565 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:36.565 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:36.565 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:36.566 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:36.566 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:36.566 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:36.584 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:36.585 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:36.585 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:36.585 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:36.585 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:36.585 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:36.585 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:36.586 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:36.586 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:36.604 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:36.605 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:36.605 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:36.605 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:36.605 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:36.605 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:36.605 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:36.605 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:36.606 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:36.624 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:36.625 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:36.625 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:36.625 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:36.625 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:36.625 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:36.625 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:36.625 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:36.626 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:36.644 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:36.644 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:36.645 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:36.645 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:36.645 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:36.645 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:36.645 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:36.645 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:36.646 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:36.704 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:36.705 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:36.705 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:36.705 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:36.705 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:36.705 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:36.705 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:36.705 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:36.706 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:36.744 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:36.745 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:36.746 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:36.746 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:36.746 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:36.747 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:36.747 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:36.748 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:36.748 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:36.749 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=204), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [161ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [161ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [162ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [202ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [204ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [204ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [204ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, deferred=Deferred@945259589(state=PENDING, result=null, callback=(continuation of Deferred@26749361 after retry RPC after error@1500154299) -> (continuation of Deferred@255784992 after retry RPC after error@189734958) -> (continuation of Deferred@1272599895 after retry RPC after error@1181756102) -> (continuation of Deferred@1047999952 after retry RPC after error@1279994976) -> (continuation of Deferred@1436727379 after retry RPC after error@314809915) -> (continuation of Deferred@362668038 after retry RPC after error@10544889) -> (continuation of Deferred@1073784478 after retry RPC after error@1320995555), errback=(continuation of Deferred@26749361 after retry RPC after error@1500154299) -> (continuation of Deferred@255784992 after retry RPC after error@189734958) -> (continuation of Deferred@1272599895 after retry RPC after error@1181756102) -> (continuation of Deferred@1047999952 after retry RPC after error@1279994976) -> (continuation of Deferred@1436727379 after retry RPC after error@314809915) -> (continuation of Deferred@362668038 after retry RPC after error@10544889) -> (continuation of Deferred@1073784478 after retry RPC after error@1320995555)))
21:04:36.749 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=204), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [161ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [161ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [162ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [202ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [204ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [204ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [204ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, deferred=Deferred@945259589(state=PENDING, result=null, callback=(continuation of Deferred@26749361 after retry RPC after error@1500154299) -> (continuation of Deferred@255784992 after retry RPC after error@189734958) -> (continuation of Deferred@1272599895 after retry RPC after error@1181756102) -> (continuation of Deferred@1047999952 after retry RPC after error@1279994976) -> (continuation of Deferred@1436727379 after retry RPC after error@314809915) -> (continuation of Deferred@362668038 after retry RPC after error@10544889) -> (continuation of Deferred@1073784478 after retry RPC after error@1320995555), errback=(continuation of Deferred@26749361 after retry RPC after error@1500154299) -> (continuation of Deferred@255784992 after retry RPC after error@189734958) -> (continuation of Deferred@1272599895 after retry RPC after error@1181756102) -> (continuation of Deferred@1047999952 after retry RPC after error@1279994976) -> (continuation of Deferred@1436727379 after retry RPC after error@314809915) -> (continuation of Deferred@362668038 after retry RPC after error@10544889) -> (continuation of Deferred@1073784478 after retry RPC after error@1320995555)))
21:04:36.764 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:36.765 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:36.765 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:36.765 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:36.765 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:36.765 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:36.765 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:36.765 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:36.766 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:36.784 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:36.784 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:36.785 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:36.785 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:36.785 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:36.785 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:36.785 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:36.785 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:36.786 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:36.804 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:36.804 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:36.805 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:36.805 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:36.805 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:36.805 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:36.805 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:36.805 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:36.806 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:36.824 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:36.824 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:36.825 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:36.825 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:36.825 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:36.825 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:36.825 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:36.825 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:36.826 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:36.845 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:36.845 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:36.845 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:36.845 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:36.845 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:36.846 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:36.846 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:36.846 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:36.846 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:36.846 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:36.847 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:36.847 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:36.864 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:36.865 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:36.865 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:36.865 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:36.865 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:36.865 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:36.865 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:36.866 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:36.866 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:36.944 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:36.944 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:36.945 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:36.945 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:36.945 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:36.945 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:36.945 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:36.945 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:36.946 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:36.984 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:36.984 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:36.985 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:36.985 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:36.985 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:36.985 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:36.985 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:36.985 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:36.986 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:36.987 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=222), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [59ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [81ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [83ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [181ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [181ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [182ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [221ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [221ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [222ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [222ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, deferred=Deferred@24717585(state=PENDING, result=null, callback=(continuation of Deferred@95016555 after retry RPC after error@377185816) -> (continuation of Deferred@61116847 after retry RPC after error@1272779115) -> (continuation of Deferred@9215221 after retry RPC after error@1292525914) -> (continuation of Deferred@1787078845 after retry RPC after error@1240574671) -> (continuation of Deferred@763292720 after retry RPC after error@861952116) -> (continuation of Deferred@2087641061 after retry RPC after error@948811125) -> (continuation of Deferred@1648939320 after retry RPC after error@1406830333), errback=(continuation of Deferred@95016555 after retry RPC after error@377185816) -> (continuation of Deferred@61116847 after retry RPC after error@1272779115) -> (continuation of Deferred@9215221 after retry RPC after error@1292525914) -> (continuation of Deferred@1787078845 after retry RPC after error@1240574671) -> (continuation of Deferred@763292720 after retry RPC after error@861952116) -> (continuation of Deferred@2087641061 after retry RPC after error@948811125) -> (continuation of Deferred@1648939320 after retry RPC after error@1406830333)))
21:04:36.987 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=222), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [59ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [81ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [83ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [181ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [181ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [182ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [221ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [221ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [222ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [222ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, deferred=Deferred@24717585(state=PENDING, result=null, callback=(continuation of Deferred@95016555 after retry RPC after error@377185816) -> (continuation of Deferred@61116847 after retry RPC after error@1272779115) -> (continuation of Deferred@9215221 after retry RPC after error@1292525914) -> (continuation of Deferred@1787078845 after retry RPC after error@1240574671) -> (continuation of Deferred@763292720 after retry RPC after error@861952116) -> (continuation of Deferred@2087641061 after retry RPC after error@948811125) -> (continuation of Deferred@1648939320 after retry RPC after error@1406830333), errback=(continuation of Deferred@95016555 after retry RPC after error@377185816) -> (continuation of Deferred@61116847 after retry RPC after error@1272779115) -> (continuation of Deferred@9215221 after retry RPC after error@1292525914) -> (continuation of Deferred@1787078845 after retry RPC after error@1240574671) -> (continuation of Deferred@763292720 after retry RPC after error@861952116) -> (continuation of Deferred@2087641061 after retry RPC after error@948811125) -> (continuation of Deferred@1648939320 after retry RPC after error@1406830333)))
21:04:37.004 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:37.004 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:37.005 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:37.005 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:37.005 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:37.005 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:37.005 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:37.005 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:37.006 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:37.024 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:37.024 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:37.025 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:37.025 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:37.025 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:37.025 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:37.025 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:37.025 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:37.026 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:37.044 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:37.044 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:37.045 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:37.045 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:37.045 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:37.045 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:37.045 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:37.045 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:37.046 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:37.064 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:37.064 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:37.065 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:37.065 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:37.065 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:37.065 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:37.065 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:37.065 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:37.066 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:37.084 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:37.084 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:37.085 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:37.085 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:37.085 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:37.085 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:37.085 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:37.085 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:37.086 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:37.104 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:37.104 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:37.105 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:37.105 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:37.105 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:37.105 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:37.105 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:37.105 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:37.106 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:37.144 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:37.144 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:37.145 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:37.145 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:37.145 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:37.145 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:37.145 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:37.145 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:37.146 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:37.147 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=142), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [141ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [141ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [142ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, deferred=Deferred@848934517(state=PENDING, result=null, callback=(continuation of Deferred@692244528 after retry RPC after error@47289907) -> (continuation of Deferred@1376908904 after retry RPC after error@759482433) -> (continuation of Deferred@314072713 after retry RPC after error@475411171) -> (continuation of Deferred@1093219767 after retry RPC after error@1243937555) -> (continuation of Deferred@448309448 after retry RPC after error@928272470) -> (continuation of Deferred@1071140742 after retry RPC after error@1380566797), errback=(continuation of Deferred@692244528 after retry RPC after error@47289907) -> (continuation of Deferred@1376908904 after retry RPC after error@759482433) -> (continuation of Deferred@314072713 after retry RPC after error@475411171) -> (continuation of Deferred@1093219767 after retry RPC after error@1243937555) -> (continuation of Deferred@448309448 after retry RPC after error@928272470) -> (continuation of Deferred@1071140742 after retry RPC after error@1380566797)))
21:04:37.147 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=142), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [141ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [141ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [142ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, deferred=Deferred@848934517(state=PENDING, result=null, callback=(continuation of Deferred@692244528 after retry RPC after error@47289907) -> (continuation of Deferred@1376908904 after retry RPC after error@759482433) -> (continuation of Deferred@314072713 after retry RPC after error@475411171) -> (continuation of Deferred@1093219767 after retry RPC after error@1243937555) -> (continuation of Deferred@448309448 after retry RPC after error@928272470) -> (continuation of Deferred@1071140742 after retry RPC after error@1380566797), errback=(continuation of Deferred@692244528 after retry RPC after error@47289907) -> (continuation of Deferred@1376908904 after retry RPC after error@759482433) -> (continuation of Deferred@314072713 after retry RPC after error@475411171) -> (continuation of Deferred@1093219767 after retry RPC after error@1243937555) -> (continuation of Deferred@448309448 after retry RPC after error@928272470) -> (continuation of Deferred@1071140742 after retry RPC after error@1380566797)))
21:04:37.164 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:37.164 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:37.165 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:37.165 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:37.165 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:37.165 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:37.166 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:37.166 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:37.166 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:37.184 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:37.184 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:37.185 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:37.185 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:37.185 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:37.185 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:37.185 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:37.185 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:37.186 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:37.204 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:37.205 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:37.205 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:37.205 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:37.205 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:37.206 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:37.206 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:37.206 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:37.207 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:37.224 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:37.225 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:37.225 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:37.225 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:37.225 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:37.225 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:37.226 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:37.226 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:37.227 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:37.244 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:37.245 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:37.245 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:37.245 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:37.245 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:37.245 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:37.246 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:37.246 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:37.246 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:37.284 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:37.285 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:37.285 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:37.285 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:37.285 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:37.285 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:37.285 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:37.285 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:37.286 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:37.364 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:37.364 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:37.365 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:37.365 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:37.365 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:37.365 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:37.365 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:37.365 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:37.366 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:37.367 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=202), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [63ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [199ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [201ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [201ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [202ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, deferred=Deferred@658622037(state=PENDING, result=null, callback=(continuation of Deferred@703046275 after retry RPC after error@115987922) -> (continuation of Deferred@582582507 after retry RPC after error@191076743) -> (continuation of Deferred@528794066 after retry RPC after error@1468780725) -> (continuation of Deferred@970697847 after retry RPC after error@786670954) -> (continuation of Deferred@399534901 after retry RPC after error@509026050) -> (continuation of Deferred@2047582379 after retry RPC after error@1232114293), errback=(continuation of Deferred@703046275 after retry RPC after error@115987922) -> (continuation of Deferred@582582507 after retry RPC after error@191076743) -> (continuation of Deferred@528794066 after retry RPC after error@1468780725) -> (continuation of Deferred@970697847 after retry RPC after error@786670954) -> (continuation of Deferred@399534901 after retry RPC after error@509026050) -> (continuation of Deferred@2047582379 after retry RPC after error@1232114293)))
21:04:37.367 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=202), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [63ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [199ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [201ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [201ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [202ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, deferred=Deferred@658622037(state=PENDING, result=null, callback=(continuation of Deferred@703046275 after retry RPC after error@115987922) -> (continuation of Deferred@582582507 after retry RPC after error@191076743) -> (continuation of Deferred@528794066 after retry RPC after error@1468780725) -> (continuation of Deferred@970697847 after retry RPC after error@786670954) -> (continuation of Deferred@399534901 after retry RPC after error@509026050) -> (continuation of Deferred@2047582379 after retry RPC after error@1232114293), errback=(continuation of Deferred@703046275 after retry RPC after error@115987922) -> (continuation of Deferred@582582507 after retry RPC after error@191076743) -> (continuation of Deferred@528794066 after retry RPC after error@1468780725) -> (continuation of Deferred@970697847 after retry RPC after error@786670954) -> (continuation of Deferred@399534901 after retry RPC after error@509026050) -> (continuation of Deferred@2047582379 after retry RPC after error@1232114293)))
21:04:37.384 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:37.384 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:37.385 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:37.385 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:37.385 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:37.385 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:37.385 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:37.385 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:37.387 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:37.405 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:37.405 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:37.405 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:37.405 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:37.405 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:37.406 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:37.406 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:37.406 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:37.407 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:37.424 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:37.425 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:37.425 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:37.425 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:37.425 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:37.425 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:37.425 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:37.425 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:37.427 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:37.444 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:37.445 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:37.445 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:37.445 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:37.445 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:37.445 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:37.445 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:37.445 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:37.446 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:37.464 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:37.465 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:37.465 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:37.465 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:37.465 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:37.465 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:37.465 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:37.465 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:37.466 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:37.484 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:37.484 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:37.485 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:37.485 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:37.485 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:37.485 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:37.485 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:37.485 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:37.486 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:37.504 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:37.505 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:37.505 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:37.505 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:37.505 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:37.505 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:37.505 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:37.505 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:37.506 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:37.624 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:37.625 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:37.625 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:37.625 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:37.625 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:37.625 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:37.625 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:37.625 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:37.626 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:37.627 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=243), Traces: [0ms] refreshing cache from master, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [3ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [4ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [21ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [22ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [22ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [23ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [23ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [23ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [24ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [41ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [42ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [42ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [43ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [44ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [61ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [62ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [63ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [64ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [81ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [83ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [101ms] refreshing cache from master, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [103ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [121ms] refreshing cache from master, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [123ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [241ms] refreshing cache from master, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [242ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [242ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [243ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [244ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, deferred=Deferred@1144326911(state=PENDING, result=null, callback=(continuation of Deferred@1701242778 after retry RPC after error@115909396) -> (continuation of Deferred@1907457791 after retry RPC after error@490725572) -> (continuation of Deferred@1337433806 after retry RPC after error@210002991) -> (continuation of Deferred@1983428038 after retry RPC after error@342049285) -> (continuation of Deferred@1741811625 after retry RPC after error@1222330976) -> (continuation of Deferred@2080264248 after retry RPC after error@1971541544) -> (continuation of Deferred@491209622 after retry RPC after error@257790484), errback=(continuation of Deferred@1701242778 after retry RPC after error@115909396) -> (continuation of Deferred@1907457791 after retry RPC after error@490725572) -> (continuation of Deferred@1337433806 after retry RPC after error@210002991) -> (continuation of Deferred@1983428038 after retry RPC after error@342049285) -> (continuation of Deferred@1741811625 after retry RPC after error@1222330976) -> (continuation of Deferred@2080264248 after retry RPC after error@1971541544) -> (continuation of Deferred@491209622 after retry RPC after error@257790484)))
21:04:37.628 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=243), Traces: [0ms] refreshing cache from master, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [3ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [4ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [21ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [22ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [22ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [23ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [23ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [23ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [24ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [41ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [42ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [42ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [43ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [44ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [61ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [62ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [63ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [64ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [81ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [83ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [101ms] refreshing cache from master, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [103ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [121ms] refreshing cache from master, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [123ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [241ms] refreshing cache from master, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [242ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [242ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [243ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [244ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, deferred=Deferred@1144326911(state=PENDING, result=null, callback=(continuation of Deferred@1701242778 after retry RPC after error@115909396) -> (continuation of Deferred@1907457791 after retry RPC after error@490725572) -> (continuation of Deferred@1337433806 after retry RPC after error@210002991) -> (continuation of Deferred@1983428038 after retry RPC after error@342049285) -> (continuation of Deferred@1741811625 after retry RPC after error@1222330976) -> (continuation of Deferred@2080264248 after retry RPC after error@1971541544) -> (continuation of Deferred@491209622 after retry RPC after error@257790484), errback=(continuation of Deferred@1701242778 after retry RPC after error@115909396) -> (continuation of Deferred@1907457791 after retry RPC after error@490725572) -> (continuation of Deferred@1337433806 after retry RPC after error@210002991) -> (continuation of Deferred@1983428038 after retry RPC after error@342049285) -> (continuation of Deferred@1741811625 after retry RPC after error@1222330976) -> (continuation of Deferred@2080264248 after retry RPC after error@1971541544) -> (continuation of Deferred@491209622 after retry RPC after error@257790484)))
21:04:37.644 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:37.645 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:37.645 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:37.645 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:37.645 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:37.645 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:37.645 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:37.645 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:37.646 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:37.664 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:37.665 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:37.665 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:37.665 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:37.665 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:37.665 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:37.665 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:37.665 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:37.666 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:37.684 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:37.684 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:37.685 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:37.685 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:37.685 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:37.685 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:37.685 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:37.685 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:37.686 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:37.704 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:37.704 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:37.705 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:37.705 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:37.705 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:37.705 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:37.705 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:37.705 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:37.706 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:37.724 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:37.724 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:37.725 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:37.725 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:37.725 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:37.725 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:37.725 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:37.725 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:37.726 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:37.744 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:37.744 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:37.745 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:37.745 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:37.745 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:37.745 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:37.745 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:37.745 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:37.746 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:37.804 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:37.805 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:37.805 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:37.805 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:37.805 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:37.806 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:37.806 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:37.806 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:37.806 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:37.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:37.891494 23569 master_proxy_rpc.cc:177] Failed to determine new leader Master: Timed out: ConnectToClusterRpc(addrs: 127.22.168.62:37239,127.22.168.61:32947,127.22.168.60:33247, num_attempts: 97) passed its deadline: Not found: no leader found: ConnectToClusterRpc(addrs: 127.22.168.62:37239,127.22.168.61:32947,127.22.168.60:33247, num_attempts: 1)
21:04:37.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:37.896961 23569 meta_cache.cc:788] Timed out: LookupRpcById { tablet: '01aa0e192c434b468ec7ed90d48563b3', attempt: 2 } failed: LookupRpcById timed out after deadline expired: LookupRpcById { tablet: '01aa0e192c434b468ec7ed90d48563b3', attempt: 2 } passed its deadline: Timed out: ConnectToClusterRpc(addrs: 127.22.168.62:37239,127.22.168.61:32947,127.22.168.60:33247, num_attempts: 97) passed its deadline: Not found: no leader found: ConnectToClusterRpc(addrs: 127.22.168.62:37239,127.22.168.61:32947,127.22.168.60:33247, num_attempts: 1)
21:04:37.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:37.899238 23569 master_proxy_rpc.cc:203] Re-attempting LookupRpcById request to leader Master (127.22.168.60:33247)
21:04:37.904 [DEBUG - pool-40-thread-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=259), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [161ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [162ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [162ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, deferred=Deferred@504508196(state=PENDING, result=null, callback=(continuation of Deferred@1767717997 after retry RPC after error@1930115291) -> (continuation of Deferred@304475895 after retry RPC after error@490838168) -> (continuation of Deferred@494887177 after retry RPC after error@932728733) -> (continuation of Deferred@175590863 after retry RPC after error@459505985) -> (continuation of Deferred@2113068045 after retry RPC after error@149127305) -> (continuation of Deferred@952806767 after retry RPC after error@1067731646) -> (continuation of Deferred@1423830003 after retry RPC after error@1076809257), errback=(continuation of Deferred@1767717997 after retry RPC after error@1930115291) -> (continuation of Deferred@304475895 after retry RPC after error@490838168) -> (continuation of Deferred@494887177 after retry RPC after error@932728733) -> (continuation of Deferred@175590863 after retry RPC after error@459505985) -> (continuation of Deferred@2113068045 after retry RPC after error@149127305) -> (continuation of Deferred@952806767 after retry RPC after error@1067731646) -> (continuation of Deferred@1423830003 after retry RPC after error@1076809257)))
21:04:37.905 [WARN - pool-40-thread-1] (HashedWheelTimer.java:716) An exception was thrown by TimerTask.
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343) ~[main/:?]
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at java.lang.Thread.run(Thread.java:748) [?:1.8.0_275]
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86)
at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106)
at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198)
at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158)
at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953)
at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923)
at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464)
at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713)
at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701)
at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788)
at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501)
at java.lang.Thread.run(Thread.java:748)
21:04:37.924 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:37.925 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:37.925 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:37.925 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:37.925 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:37.925 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:37.925 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:37.925 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:37.926 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:37.944 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:37.945 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:37.945 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:37.945 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:37.945 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:37.945 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:37.946 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:37.946 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:37.947 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:37.964 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:37.965 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:37.965 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:37.965 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:37.965 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:37.965 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:37.965 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:37.965 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:37.966 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:37.984 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:37.984 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:37.985 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:37.985 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:37.985 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:37.985 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:37.985 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:37.986 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:37.986 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:38.004 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:38.004 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:38.005 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:38.005 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:38.005 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:38.005 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:38.005 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:38.005 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:38.006 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:38.044 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:38.045 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:38.045 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:38.045 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:38.045 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:38.045 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:38.045 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:38.046 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:38.046 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:38.064 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:38.064 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:38.065 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:38.065 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:38.065 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:38.065 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:38.065 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:38.065 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:38.066 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:38.144 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:38.145 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:38.145 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:38.145 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:38.146 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:38.146 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:38.146 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:38.146 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:38.146 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:38.147 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=222), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [141ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [141ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [142ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [222ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [222ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [222ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [222ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, deferred=Deferred@1651111955(state=PENDING, result=null, callback=(continuation of Deferred@2079492322 after retry RPC after error@668014213) -> (continuation of Deferred@1628653951 after retry RPC after error@1285004334) -> (continuation of Deferred@600680646 after retry RPC after error@201848580) -> (continuation of Deferred@1998851914 after retry RPC after error@119704428) -> (continuation of Deferred@1239910614 after retry RPC after error@2062285113) -> (continuation of Deferred@761460593 after retry RPC after error@1049129050) -> (continuation of Deferred@1272770149 after retry RPC after error@1859904534), errback=(continuation of Deferred@2079492322 after retry RPC after error@668014213) -> (continuation of Deferred@1628653951 after retry RPC after error@1285004334) -> (continuation of Deferred@600680646 after retry RPC after error@201848580) -> (continuation of Deferred@1998851914 after retry RPC after error@119704428) -> (continuation of Deferred@1239910614 after retry RPC after error@2062285113) -> (continuation of Deferred@761460593 after retry RPC after error@1049129050) -> (continuation of Deferred@1272770149 after retry RPC after error@1859904534)))
21:04:38.147 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=222), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [141ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [141ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [142ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [222ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [222ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [222ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [222ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, deferred=Deferred@1651111955(state=PENDING, result=null, callback=(continuation of Deferred@2079492322 after retry RPC after error@668014213) -> (continuation of Deferred@1628653951 after retry RPC after error@1285004334) -> (continuation of Deferred@600680646 after retry RPC after error@201848580) -> (continuation of Deferred@1998851914 after retry RPC after error@119704428) -> (continuation of Deferred@1239910614 after retry RPC after error@2062285113) -> (continuation of Deferred@761460593 after retry RPC after error@1049129050) -> (continuation of Deferred@1272770149 after retry RPC after error@1859904534), errback=(continuation of Deferred@2079492322 after retry RPC after error@668014213) -> (continuation of Deferred@1628653951 after retry RPC after error@1285004334) -> (continuation of Deferred@600680646 after retry RPC after error@201848580) -> (continuation of Deferred@1998851914 after retry RPC after error@119704428) -> (continuation of Deferred@1239910614 after retry RPC after error@2062285113) -> (continuation of Deferred@761460593 after retry RPC after error@1049129050) -> (continuation of Deferred@1272770149 after retry RPC after error@1859904534)))
21:04:38.164 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:38.165 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:38.165 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:38.165 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:38.165 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:38.166 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:38.166 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:38.166 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:38.166 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:38.184 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:38.185 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:38.185 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:38.185 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:38.185 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:38.185 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:38.185 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:38.185 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:38.186 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:38.204 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:38.204 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:38.205 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:38.205 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:38.205 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:38.205 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:38.206 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:38.206 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:38.207 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:38.224 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:38.224 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:38.225 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:38.225 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:38.225 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:38.225 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:38.225 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:38.225 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:38.226 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:38.244 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:38.244 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:38.245 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:38.245 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:38.245 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:38.245 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:38.245 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:38.245 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:38.246 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:38.264 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:38.264 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:38.265 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:38.265 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:38.265 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:38.265 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:38.265 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:38.265 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:38.266 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:38.324 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:38.324 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:38.325 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:38.325 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:38.325 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:38.326 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:38.326 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:38.326 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:38.326 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:38.364 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:38.365 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:38.365 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:38.365 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:38.365 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:38.365 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:38.365 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:38.366 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:38.366 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:38.367 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=202), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [43ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [161ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [162ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [162ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [201ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [201ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [202ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, deferred=Deferred@2087117176(state=PENDING, result=null, callback=(continuation of Deferred@1989074460 after retry RPC after error@1753099130) -> (continuation of Deferred@402351926 after retry RPC after error@1558002064) -> (continuation of Deferred@1554211369 after retry RPC after error@1168090940) -> (continuation of Deferred@152152686 after retry RPC after error@1010111333) -> (continuation of Deferred@2067832032 after retry RPC after error@1138651891) -> (continuation of Deferred@1201201724 after retry RPC after error@1394769809) -> (continuation of Deferred@754601609 after retry RPC after error@470018063), errback=(continuation of Deferred@1989074460 after retry RPC after error@1753099130) -> (continuation of Deferred@402351926 after retry RPC after error@1558002064) -> (continuation of Deferred@1554211369 after retry RPC after error@1168090940) -> (continuation of Deferred@152152686 after retry RPC after error@1010111333) -> (continuation of Deferred@2067832032 after retry RPC after error@1138651891) -> (continuation of Deferred@1201201724 after retry RPC after error@1394769809) -> (continuation of Deferred@754601609 after retry RPC after error@470018063)))
21:04:38.367 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=202), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [43ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [161ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [162ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [162ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [201ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [201ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [202ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, deferred=Deferred@2087117176(state=PENDING, result=null, callback=(continuation of Deferred@1989074460 after retry RPC after error@1753099130) -> (continuation of Deferred@402351926 after retry RPC after error@1558002064) -> (continuation of Deferred@1554211369 after retry RPC after error@1168090940) -> (continuation of Deferred@152152686 after retry RPC after error@1010111333) -> (continuation of Deferred@2067832032 after retry RPC after error@1138651891) -> (continuation of Deferred@1201201724 after retry RPC after error@1394769809) -> (continuation of Deferred@754601609 after retry RPC after error@470018063), errback=(continuation of Deferred@1989074460 after retry RPC after error@1753099130) -> (continuation of Deferred@402351926 after retry RPC after error@1558002064) -> (continuation of Deferred@1554211369 after retry RPC after error@1168090940) -> (continuation of Deferred@152152686 after retry RPC after error@1010111333) -> (continuation of Deferred@2067832032 after retry RPC after error@1138651891) -> (continuation of Deferred@1201201724 after retry RPC after error@1394769809) -> (continuation of Deferred@754601609 after retry RPC after error@470018063)))
21:04:38.384 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:38.385 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:38.385 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:38.385 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:38.385 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:38.385 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:38.385 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:38.385 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:38.386 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:38.404 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:38.404 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:38.405 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:38.405 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:38.405 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:38.405 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:38.405 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:38.405 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:38.406 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:38.424 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:38.424 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:38.425 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:38.425 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:38.425 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:38.425 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:38.425 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:38.425 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:38.426 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:38.444 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:38.444 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:38.445 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:38.445 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:38.445 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:38.445 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:38.445 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:38.445 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:38.446 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:38.464 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:38.464 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:38.465 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:38.465 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:38.465 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:38.465 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:38.465 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:38.465 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:38.466 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:38.484 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:38.484 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:38.485 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:38.485 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:38.485 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:38.485 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:38.485 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:38.485 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:38.486 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:38.544 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:38.544 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:38.545 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:38.545 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:38.545 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:38.545 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:38.545 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:38.545 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:38.546 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:38.546 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=162), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [99ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [161ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [161ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [162ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, deferred=Deferred@206192336(state=PENDING, result=null, callback=(continuation of Deferred@597429134 after retry RPC after error@495138677) -> (continuation of Deferred@470617759 after retry RPC after error@1097604976) -> (continuation of Deferred@49261736 after retry RPC after error@114494239) -> (continuation of Deferred@1918796972 after retry RPC after error@1407750664) -> (continuation of Deferred@740159025 after retry RPC after error@839418718) -> (continuation of Deferred@880089687 after retry RPC after error@597341699), errback=(continuation of Deferred@597429134 after retry RPC after error@495138677) -> (continuation of Deferred@470617759 after retry RPC after error@1097604976) -> (continuation of Deferred@49261736 after retry RPC after error@114494239) -> (continuation of Deferred@1918796972 after retry RPC after error@1407750664) -> (continuation of Deferred@740159025 after retry RPC after error@839418718) -> (continuation of Deferred@880089687 after retry RPC after error@597341699)))
21:04:38.547 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=162), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [99ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [161ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [161ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [162ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, deferred=Deferred@206192336(state=PENDING, result=null, callback=(continuation of Deferred@597429134 after retry RPC after error@495138677) -> (continuation of Deferred@470617759 after retry RPC after error@1097604976) -> (continuation of Deferred@49261736 after retry RPC after error@114494239) -> (continuation of Deferred@1918796972 after retry RPC after error@1407750664) -> (continuation of Deferred@740159025 after retry RPC after error@839418718) -> (continuation of Deferred@880089687 after retry RPC after error@597341699), errback=(continuation of Deferred@597429134 after retry RPC after error@495138677) -> (continuation of Deferred@470617759 after retry RPC after error@1097604976) -> (continuation of Deferred@49261736 after retry RPC after error@114494239) -> (continuation of Deferred@1918796972 after retry RPC after error@1407750664) -> (continuation of Deferred@740159025 after retry RPC after error@839418718) -> (continuation of Deferred@880089687 after retry RPC after error@597341699)))
21:04:38.564 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:38.565 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:38.565 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:38.565 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:38.565 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:38.565 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:38.565 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:38.565 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:38.566 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:38.584 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:38.584 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:38.585 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:38.585 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:38.585 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:38.585 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:38.585 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:38.585 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:38.586 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:38.604 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:38.604 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:38.605 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:38.605 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:38.605 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:38.605 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:38.605 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:38.605 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:38.606 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:38.624 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:38.624 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:38.625 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:38.625 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:38.625 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:38.625 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:38.625 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:38.625 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:38.626 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:38.644 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:38.644 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:38.645 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:38.645 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:38.645 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:38.645 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:38.645 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:38.645 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:38.646 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:38.664 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:38.664 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:38.665 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:38.665 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:38.665 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:38.665 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:38.665 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:38.665 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:38.666 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:38.704 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:38.705 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:38.705 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:38.705 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:38.705 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:38.705 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:38.705 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:38.705 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:38.706 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:38.804 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:38.804 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:38.805 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:38.805 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:38.805 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:38.805 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:38.805 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:38.805 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:38.806 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:38.806 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=242), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [141ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [141ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [142ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [239ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [241ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [241ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [241ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [242ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, deferred=Deferred@1773627748(state=PENDING, result=null, callback=(continuation of Deferred@1587575397 after retry RPC after error@346689884) -> (continuation of Deferred@1555671464 after retry RPC after error@244778770) -> (continuation of Deferred@1396254711 after retry RPC after error@392031293) -> (continuation of Deferred@112780183 after retry RPC after error@815909829) -> (continuation of Deferred@717051065 after retry RPC after error@1392077271) -> (continuation of Deferred@320642116 after retry RPC after error@769491457) -> (continuation of Deferred@1829045151 after retry RPC after error@1949005881), errback=(continuation of Deferred@1587575397 after retry RPC after error@346689884) -> (continuation of Deferred@1555671464 after retry RPC after error@244778770) -> (continuation of Deferred@1396254711 after retry RPC after error@392031293) -> (continuation of Deferred@112780183 after retry RPC after error@815909829) -> (continuation of Deferred@717051065 after retry RPC after error@1392077271) -> (continuation of Deferred@320642116 after retry RPC after error@769491457) -> (continuation of Deferred@1829045151 after retry RPC after error@1949005881)))
21:04:38.807 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=242), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [141ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [141ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [142ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [239ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [241ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [241ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [241ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [242ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, deferred=Deferred@1773627748(state=PENDING, result=null, callback=(continuation of Deferred@1587575397 after retry RPC after error@346689884) -> (continuation of Deferred@1555671464 after retry RPC after error@244778770) -> (continuation of Deferred@1396254711 after retry RPC after error@392031293) -> (continuation of Deferred@112780183 after retry RPC after error@815909829) -> (continuation of Deferred@717051065 after retry RPC after error@1392077271) -> (continuation of Deferred@320642116 after retry RPC after error@769491457) -> (continuation of Deferred@1829045151 after retry RPC after error@1949005881), errback=(continuation of Deferred@1587575397 after retry RPC after error@346689884) -> (continuation of Deferred@1555671464 after retry RPC after error@244778770) -> (continuation of Deferred@1396254711 after retry RPC after error@392031293) -> (continuation of Deferred@112780183 after retry RPC after error@815909829) -> (continuation of Deferred@717051065 after retry RPC after error@1392077271) -> (continuation of Deferred@320642116 after retry RPC after error@769491457) -> (continuation of Deferred@1829045151 after retry RPC after error@1949005881)))
21:04:38.825 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:38.826 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:38.826 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:38.827 [INFO - pool-40-thread-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:38.829 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:38.830 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:38.830 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:38.830 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:38.831 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:38.844 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:38.844 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:38.845 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:38.845 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:38.845 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:38.845 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:38.845 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:38.846 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:38.846 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:38.864 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:38.865 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:38.865 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:38.865 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:38.865 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:38.865 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:38.865 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:38.865 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:38.866 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:38.884 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:38.885 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:38.885 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:38.885 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:38.885 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:38.885 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:38.885 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:38.885 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:38.886 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:38.904 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:38.904 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:38.905 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:38.905 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:38.905 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:38.905 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:38.905 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:38.906 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:38.906 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:38.924 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:38.924 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:38.925 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:38.925 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:38.925 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:38.925 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:38.925 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:38.925 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:38.926 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:38.944 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:38.944 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:38.945 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:38.945 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:38.945 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:38.945 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:38.945 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:38.945 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:38.946 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:39.084 [DEBUG - pool-40-thread-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=259), Traces: [0ms] refreshing cache from master, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [3ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [5ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [5ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [6ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [7ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, deferred=Deferred@888481516(state=PENDING, result=null, callback=(continuation of Deferred@1411317153 after retry RPC after error@1792897448) -> (continuation of Deferred@7588833 after retry RPC after error@489443141) -> (continuation of Deferred@1056725684 after retry RPC after error@1068313842) -> (continuation of Deferred@555734053 after retry RPC after error@1589043492) -> (continuation of Deferred@46344636 after retry RPC after error@1423921066) -> (continuation of Deferred@197003240 after retry RPC after error@1651009658) -> (continuation of Deferred@2038248258 after retry RPC after error@1467056600), errback=(continuation of Deferred@1411317153 after retry RPC after error@1792897448) -> (continuation of Deferred@7588833 after retry RPC after error@489443141) -> (continuation of Deferred@1056725684 after retry RPC after error@1068313842) -> (continuation of Deferred@555734053 after retry RPC after error@1589043492) -> (continuation of Deferred@46344636 after retry RPC after error@1423921066) -> (continuation of Deferred@197003240 after retry RPC after error@1651009658) -> (continuation of Deferred@2038248258 after retry RPC after error@1467056600)))
21:04:39.085 [WARN - pool-40-thread-1] (HashedWheelTimer.java:716) An exception was thrown by TimerTask.
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343) ~[main/:?]
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at java.lang.Thread.run(Thread.java:748) [?:1.8.0_275]
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86)
at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106)
at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198)
at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158)
at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953)
at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923)
at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464)
at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713)
at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701)
at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788)
at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501)
at java.lang.Thread.run(Thread.java:748)
21:04:39.104 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:39.105 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:39.105 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:39.105 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:39.105 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:39.105 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:39.106 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:39.106 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:39.106 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:39.124 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:39.125 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:39.125 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:39.125 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:39.125 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:39.125 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:39.125 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:39.125 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:39.126 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:39.144 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:39.144 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:39.145 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:39.145 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:39.145 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:39.145 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:39.145 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:39.145 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:39.146 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:39.164 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:39.165 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:39.165 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:39.165 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:39.165 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:39.165 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:39.165 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:39.165 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:39.166 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:39.184 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:39.185 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:39.185 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:39.185 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:39.185 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:39.185 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:39.186 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:39.186 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:39.190 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:39.204 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:39.205 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:39.206 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:39.206 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:39.206 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:39.206 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:39.206 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:39.206 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:39.207 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:39.224 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:39.224 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:39.225 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:39.225 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:39.225 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:39.225 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:39.225 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:39.225 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:39.226 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:39.264 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:39.264 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:39.265 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:39.265 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:39.265 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:39.265 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:39.265 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:39.265 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:39.266 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:39.267 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=162), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [86ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [86ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [103ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [119ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [161ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [161ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [162ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, deferred=Deferred@854357430(state=PENDING, result=null, callback=(continuation of Deferred@565742277 after retry RPC after error@1818162228) -> (continuation of Deferred@101321659 after retry RPC after error@1469746953) -> (continuation of Deferred@848419229 after retry RPC after error@695563439) -> (continuation of Deferred@2138371561 after retry RPC after error@2090196937) -> (continuation of Deferred@1081392375 after retry RPC after error@1109817434) -> (continuation of Deferred@438638154 after retry RPC after error@1218246992) -> (continuation of Deferred@761948080 after retry RPC after error@1560227567), errback=(continuation of Deferred@565742277 after retry RPC after error@1818162228) -> (continuation of Deferred@101321659 after retry RPC after error@1469746953) -> (continuation of Deferred@848419229 after retry RPC after error@695563439) -> (continuation of Deferred@2138371561 after retry RPC after error@2090196937) -> (continuation of Deferred@1081392375 after retry RPC after error@1109817434) -> (continuation of Deferred@438638154 after retry RPC after error@1218246992) -> (continuation of Deferred@761948080 after retry RPC after error@1560227567)))
21:04:39.267 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=162), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [86ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [86ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [103ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [119ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [161ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [161ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [162ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, deferred=Deferred@854357430(state=PENDING, result=null, callback=(continuation of Deferred@565742277 after retry RPC after error@1818162228) -> (continuation of Deferred@101321659 after retry RPC after error@1469746953) -> (continuation of Deferred@848419229 after retry RPC after error@695563439) -> (continuation of Deferred@2138371561 after retry RPC after error@2090196937) -> (continuation of Deferred@1081392375 after retry RPC after error@1109817434) -> (continuation of Deferred@438638154 after retry RPC after error@1218246992) -> (continuation of Deferred@761948080 after retry RPC after error@1560227567), errback=(continuation of Deferred@565742277 after retry RPC after error@1818162228) -> (continuation of Deferred@101321659 after retry RPC after error@1469746953) -> (continuation of Deferred@848419229 after retry RPC after error@695563439) -> (continuation of Deferred@2138371561 after retry RPC after error@2090196937) -> (continuation of Deferred@1081392375 after retry RPC after error@1109817434) -> (continuation of Deferred@438638154 after retry RPC after error@1218246992) -> (continuation of Deferred@761948080 after retry RPC after error@1560227567)))
21:04:39.284 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:39.284 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:39.285 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:39.285 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:39.285 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:39.285 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:39.285 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:39.285 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:39.286 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:39.304 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:39.304 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:39.305 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:39.305 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:39.305 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:39.305 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:39.305 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:39.305 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:39.306 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:39.324 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:39.325 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:39.325 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:39.325 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:39.325 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:39.325 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:39.326 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:39.326 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:39.326 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:39.344 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:39.345 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:39.345 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:39.345 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:39.346 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:39.346 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:39.346 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:39.346 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:39.347 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:39.364 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:39.365 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:39.365 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:39.365 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:39.365 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:39.365 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:39.365 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:39.365 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:39.367 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:39.404 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:39.405 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:39.405 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:39.405 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:39.405 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:39.405 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:39.405 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:39.405 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:39.406 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:39.445 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:39.445 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:39.445 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:39.446 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:39.446 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:39.446 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:39.446 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:39.446 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:39.447 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:39.505 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:39.505 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:39.505 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:39.505 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:39.505 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:39.506 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:39.506 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:39.506 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:39.507 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:39.507 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=222), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [63ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [64ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [162ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [162ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [163ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [163ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [222ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [222ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [222ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [223ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, deferred=Deferred@1513208655(state=PENDING, result=null, callback=(continuation of Deferred@1809823108 after retry RPC after error@82695589) -> (continuation of Deferred@240673064 after retry RPC after error@1499748463) -> (continuation of Deferred@1325512018 after retry RPC after error@623575530) -> (continuation of Deferred@1748464966 after retry RPC after error@334233359) -> (continuation of Deferred@1359115013 after retry RPC after error@744164510) -> (continuation of Deferred@2132136214 after retry RPC after error@860510844) -> (continuation of Deferred@1905429835 after retry RPC after error@89739291), errback=(continuation of Deferred@1809823108 after retry RPC after error@82695589) -> (continuation of Deferred@240673064 after retry RPC after error@1499748463) -> (continuation of Deferred@1325512018 after retry RPC after error@623575530) -> (continuation of Deferred@1748464966 after retry RPC after error@334233359) -> (continuation of Deferred@1359115013 after retry RPC after error@744164510) -> (continuation of Deferred@2132136214 after retry RPC after error@860510844) -> (continuation of Deferred@1905429835 after retry RPC after error@89739291)))
21:04:39.508 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=222), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [63ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [64ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [162ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [162ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [163ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [163ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [222ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [222ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [222ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [223ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, deferred=Deferred@1513208655(state=PENDING, result=null, callback=(continuation of Deferred@1809823108 after retry RPC after error@82695589) -> (continuation of Deferred@240673064 after retry RPC after error@1499748463) -> (continuation of Deferred@1325512018 after retry RPC after error@623575530) -> (continuation of Deferred@1748464966 after retry RPC after error@334233359) -> (continuation of Deferred@1359115013 after retry RPC after error@744164510) -> (continuation of Deferred@2132136214 after retry RPC after error@860510844) -> (continuation of Deferred@1905429835 after retry RPC after error@89739291), errback=(continuation of Deferred@1809823108 after retry RPC after error@82695589) -> (continuation of Deferred@240673064 after retry RPC after error@1499748463) -> (continuation of Deferred@1325512018 after retry RPC after error@623575530) -> (continuation of Deferred@1748464966 after retry RPC after error@334233359) -> (continuation of Deferred@1359115013 after retry RPC after error@744164510) -> (continuation of Deferred@2132136214 after retry RPC after error@860510844) -> (continuation of Deferred@1905429835 after retry RPC after error@89739291)))
21:04:39.524 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:39.525 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:39.525 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:39.525 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:39.525 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:39.525 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:39.525 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:39.526 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:39.526 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:39.544 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:39.545 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:39.545 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:39.545 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:39.545 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:39.545 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:39.545 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:39.546 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:39.547 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:39.564 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:39.565 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:39.565 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:39.565 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:39.566 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:39.566 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:39.566 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:39.566 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:39.567 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:39.584 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:39.585 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:39.585 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:39.585 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:39.585 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:39.585 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:39.586 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:39.586 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:39.586 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:39.604 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:39.605 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:39.605 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:39.605 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:39.605 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:39.605 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:39.606 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:39.606 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:39.606 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:39.624 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:39.625 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:39.625 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:39.625 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:39.625 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:39.625 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:39.625 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:39.625 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:39.626 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:39.644 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:39.644 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:39.645 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:39.645 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:39.645 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:39.645 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:39.645 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:39.645 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:39.646 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:39.744 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:39.745 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:39.745 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:39.745 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:39.745 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:39.745 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:39.746 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:39.746 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:39.746 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:39.747 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:39.747 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:39.747 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:39.748 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=223), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [221ms] refreshing cache from master, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [222ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [223ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [223ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [224ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, deferred=Deferred@1301002659(state=PENDING, result=null, callback=(continuation of Deferred@1099682037 after retry RPC after error@1360207563) -> (continuation of Deferred@1655544544 after retry RPC after error@1920692663) -> (continuation of Deferred@1835539317 after retry RPC after error@608478549) -> (continuation of Deferred@707805607 after retry RPC after error@66026399) -> (continuation of Deferred@787577636 after retry RPC after error@798887330) -> (continuation of Deferred@118185103 after retry RPC after error@2134961266) -> (continuation of Deferred@1515032805 after retry RPC after error@98661651), errback=(continuation of Deferred@1099682037 after retry RPC after error@1360207563) -> (continuation of Deferred@1655544544 after retry RPC after error@1920692663) -> (continuation of Deferred@1835539317 after retry RPC after error@608478549) -> (continuation of Deferred@707805607 after retry RPC after error@66026399) -> (continuation of Deferred@787577636 after retry RPC after error@798887330) -> (continuation of Deferred@118185103 after retry RPC after error@2134961266) -> (continuation of Deferred@1515032805 after retry RPC after error@98661651)))
21:04:39.749 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=223), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [221ms] refreshing cache from master, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [222ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [223ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [223ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [224ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, deferred=Deferred@1301002659(state=PENDING, result=null, callback=(continuation of Deferred@1099682037 after retry RPC after error@1360207563) -> (continuation of Deferred@1655544544 after retry RPC after error@1920692663) -> (continuation of Deferred@1835539317 after retry RPC after error@608478549) -> (continuation of Deferred@707805607 after retry RPC after error@66026399) -> (continuation of Deferred@787577636 after retry RPC after error@798887330) -> (continuation of Deferred@118185103 after retry RPC after error@2134961266) -> (continuation of Deferred@1515032805 after retry RPC after error@98661651), errback=(continuation of Deferred@1099682037 after retry RPC after error@1360207563) -> (continuation of Deferred@1655544544 after retry RPC after error@1920692663) -> (continuation of Deferred@1835539317 after retry RPC after error@608478549) -> (continuation of Deferred@707805607 after retry RPC after error@66026399) -> (continuation of Deferred@787577636 after retry RPC after error@798887330) -> (continuation of Deferred@118185103 after retry RPC after error@2134961266) -> (continuation of Deferred@1515032805 after retry RPC after error@98661651)))
21:04:39.764 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:39.765 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:39.765 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:39.765 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:39.765 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:39.765 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:39.766 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:39.766 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:39.766 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:39.784 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:39.785 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:39.785 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:39.785 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:39.785 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:39.785 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:39.786 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:39.786 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:39.786 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:39.804 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:39.805 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:39.805 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:39.805 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:39.805 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:39.805 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:39.805 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:39.805 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:39.806 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:39.824 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:39.825 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:39.825 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:39.825 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:39.825 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:39.825 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:39.825 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:39.825 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:39.826 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:39.844 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:39.845 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:39.845 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:39.845 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:39.845 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:39.845 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:39.845 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:39.845 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:39.846 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:39.884 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:39.885 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:39.885 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:39.885 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:39.885 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:39.885 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:39.885 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:39.886 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:39.886 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:39.904 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:39.905 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:39.905 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:39.905 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:39.905 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:39.905 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:39.905 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:39.905 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:39.906 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:39.907 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=142), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [141ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [141ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [142ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, deferred=Deferred@1794030876(state=PENDING, result=null, callback=(continuation of Deferred@1934612865 after retry RPC after error@2021343095) -> (continuation of Deferred@444120677 after retry RPC after error@98690667) -> (continuation of Deferred@149922951 after retry RPC after error@693019242) -> (continuation of Deferred@1981463216 after retry RPC after error@2084751526) -> (continuation of Deferred@167546933 after retry RPC after error@1442659986) -> (continuation of Deferred@1980875175 after retry RPC after error@477900859), errback=(continuation of Deferred@1934612865 after retry RPC after error@2021343095) -> (continuation of Deferred@444120677 after retry RPC after error@98690667) -> (continuation of Deferred@149922951 after retry RPC after error@693019242) -> (continuation of Deferred@1981463216 after retry RPC after error@2084751526) -> (continuation of Deferred@167546933 after retry RPC after error@1442659986) -> (continuation of Deferred@1980875175 after retry RPC after error@477900859)))
21:04:39.907 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=142), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [141ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [141ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [142ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, deferred=Deferred@1794030876(state=PENDING, result=null, callback=(continuation of Deferred@1934612865 after retry RPC after error@2021343095) -> (continuation of Deferred@444120677 after retry RPC after error@98690667) -> (continuation of Deferred@149922951 after retry RPC after error@693019242) -> (continuation of Deferred@1981463216 after retry RPC after error@2084751526) -> (continuation of Deferred@167546933 after retry RPC after error@1442659986) -> (continuation of Deferred@1980875175 after retry RPC after error@477900859), errback=(continuation of Deferred@1934612865 after retry RPC after error@2021343095) -> (continuation of Deferred@444120677 after retry RPC after error@98690667) -> (continuation of Deferred@149922951 after retry RPC after error@693019242) -> (continuation of Deferred@1981463216 after retry RPC after error@2084751526) -> (continuation of Deferred@167546933 after retry RPC after error@1442659986) -> (continuation of Deferred@1980875175 after retry RPC after error@477900859)))
21:04:39.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:39.919106 24071 master_runner.cc:186] Error getting master registration for 127.22.168.62:37239: Network error: Client connection negotiation failed: client connection to 127.22.168.62:37239: connect: Connection refused (error 111),
21:04:39.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:39.919310 24071 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
21:04:39.924 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:39.924 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:39.925 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:39.925 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:39.925 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:39.925 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:39.925 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:39.925 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:39.926 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:39.944 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:39.944 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:39.945 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:39.945 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:39.945 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:39.945 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:39.945 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:39.945 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:39.946 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:39.964 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:39.964 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:39.965 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:39.965 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:39.965 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:39.965 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:39.965 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:39.965 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:39.966 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:39.984 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:39.984 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:39.985 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:39.985 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:39.985 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:39.985 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:39.985 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:39.985 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:39.986 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:40.004 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:40.004 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:40.005 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:40.005 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:40.005 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:40.005 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:40.005 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:40.005 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:40.006 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:40.025 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:40.025 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:40.026 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:40.026 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:40.026 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:40.026 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:40.026 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:40.026 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:40.027 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:40.044 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:40.045 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:40.045 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:40.045 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:40.045 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:40.046 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:40.046 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:40.046 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:40.047 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:40.144 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:40.144 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:40.145 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:40.145 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:40.145 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:40.145 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:40.145 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:40.145 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:40.146 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:40.147 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=222), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [79ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [100ms] refreshing cache from master, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [103ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [221ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [221ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [222ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [222ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, deferred=Deferred@1045294152(state=PENDING, result=null, callback=(continuation of Deferred@826220337 after retry RPC after error@1027643922) -> (continuation of Deferred@1909510638 after retry RPC after error@548297776) -> (continuation of Deferred@1763772803 after retry RPC after error@1342742207) -> (continuation of Deferred@664828657 after retry RPC after error@1975381481) -> (continuation of Deferred@347185100 after retry RPC after error@14157443) -> (continuation of Deferred@613127067 after retry RPC after error@1053028422) -> (continuation of Deferred@638863277 after retry RPC after error@1723376114), errback=(continuation of Deferred@826220337 after retry RPC after error@1027643922) -> (continuation of Deferred@1909510638 after retry RPC after error@548297776) -> (continuation of Deferred@1763772803 after retry RPC after error@1342742207) -> (continuation of Deferred@664828657 after retry RPC after error@1975381481) -> (continuation of Deferred@347185100 after retry RPC after error@14157443) -> (continuation of Deferred@613127067 after retry RPC after error@1053028422) -> (continuation of Deferred@638863277 after retry RPC after error@1723376114)))
21:04:40.148 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=222), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [79ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [100ms] refreshing cache from master, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [103ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [221ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [221ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [222ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [222ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, deferred=Deferred@1045294152(state=PENDING, result=null, callback=(continuation of Deferred@826220337 after retry RPC after error@1027643922) -> (continuation of Deferred@1909510638 after retry RPC after error@548297776) -> (continuation of Deferred@1763772803 after retry RPC after error@1342742207) -> (continuation of Deferred@664828657 after retry RPC after error@1975381481) -> (continuation of Deferred@347185100 after retry RPC after error@14157443) -> (continuation of Deferred@613127067 after retry RPC after error@1053028422) -> (continuation of Deferred@638863277 after retry RPC after error@1723376114), errback=(continuation of Deferred@826220337 after retry RPC after error@1027643922) -> (continuation of Deferred@1909510638 after retry RPC after error@548297776) -> (continuation of Deferred@1763772803 after retry RPC after error@1342742207) -> (continuation of Deferred@664828657 after retry RPC after error@1975381481) -> (continuation of Deferred@347185100 after retry RPC after error@14157443) -> (continuation of Deferred@613127067 after retry RPC after error@1053028422) -> (continuation of Deferred@638863277 after retry RPC after error@1723376114)))
21:04:40.164 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:40.164 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:40.165 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:40.165 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:40.165 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:40.165 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:40.165 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:40.165 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:40.166 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:40.184 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:40.184 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:40.185 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:40.185 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:40.185 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:40.185 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:40.185 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:40.185 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:40.186 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:40.204 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:40.204 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:40.205 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:40.205 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:40.205 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:40.205 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:40.205 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:40.205 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:40.206 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:40.224 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:40.224 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:40.225 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:40.225 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:40.225 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:40.225 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:40.225 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:40.225 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:40.226 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:40.244 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:40.244 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:40.245 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:40.245 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:40.245 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:40.245 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:40.245 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:40.245 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:40.246 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:40.284 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:40.284 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:40.285 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:40.285 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:40.285 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:40.285 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:40.285 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:40.285 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:40.286 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:40.324 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:40.325 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:40.325 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:40.325 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:40.325 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:40.325 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:40.325 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:40.325 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:40.326 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:40.344 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:40.344 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:40.345 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:40.345 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:40.345 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:40.345 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:40.345 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:40.345 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:40.346 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:40.364 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:40.364 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:40.365 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:40.365 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:40.365 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:40.365 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:40.365 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:40.366 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:40.366 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:40.367 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=9, TimeoutTracker(timeout=250, elapsed=202), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [19ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [39ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [59ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [79ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [161ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [161ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [162ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [181ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [181ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [182ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [201ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [201ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [202ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, deferred=Deferred@1416636891(state=PENDING, result=null, callback=(continuation of Deferred@1936218104 after retry RPC after error@2062119449) -> (continuation of Deferred@447726905 after retry RPC after error@1329862982) -> (continuation of Deferred@2116178113 after retry RPC after error@1332382886) -> (continuation of Deferred@721952383 after retry RPC after error@1156168817) -> (continuation of Deferred@681658485 after retry RPC after error@859172475) -> (continuation of Deferred@1487271300 after retry RPC after error@1369882837) -> (continuation of Deferred@1944504436 after retry RPC after error@1866245701) -> (continuation of Deferred@407009267 after retry RPC after error@2139470918), errback=(continuation of Deferred@1936218104 after retry RPC after error@2062119449) -> (continuation of Deferred@447726905 after retry RPC after error@1329862982) -> (continuation of Deferred@2116178113 after retry RPC after error@1332382886) -> (continuation of Deferred@721952383 after retry RPC after error@1156168817) -> (continuation of Deferred@681658485 after retry RPC after error@859172475) -> (continuation of Deferred@1487271300 after retry RPC after error@1369882837) -> (continuation of Deferred@1944504436 after retry RPC after error@1866245701) -> (continuation of Deferred@407009267 after retry RPC after error@2139470918)))
21:04:40.367 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=9, TimeoutTracker(timeout=250, elapsed=202), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [19ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [39ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [59ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [79ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [161ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [161ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [162ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [181ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [181ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [182ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [201ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [201ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [202ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, deferred=Deferred@1416636891(state=PENDING, result=null, callback=(continuation of Deferred@1936218104 after retry RPC after error@2062119449) -> (continuation of Deferred@447726905 after retry RPC after error@1329862982) -> (continuation of Deferred@2116178113 after retry RPC after error@1332382886) -> (continuation of Deferred@721952383 after retry RPC after error@1156168817) -> (continuation of Deferred@681658485 after retry RPC after error@859172475) -> (continuation of Deferred@1487271300 after retry RPC after error@1369882837) -> (continuation of Deferred@1944504436 after retry RPC after error@1866245701) -> (continuation of Deferred@407009267 after retry RPC after error@2139470918), errback=(continuation of Deferred@1936218104 after retry RPC after error@2062119449) -> (continuation of Deferred@447726905 after retry RPC after error@1329862982) -> (continuation of Deferred@2116178113 after retry RPC after error@1332382886) -> (continuation of Deferred@721952383 after retry RPC after error@1156168817) -> (continuation of Deferred@681658485 after retry RPC after error@859172475) -> (continuation of Deferred@1487271300 after retry RPC after error@1369882837) -> (continuation of Deferred@1944504436 after retry RPC after error@1866245701) -> (continuation of Deferred@407009267 after retry RPC after error@2139470918)))
21:04:40.384 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:40.384 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:40.385 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:40.385 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:40.385 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:40.385 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:40.385 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:40.386 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:40.386 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:40.404 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:40.404 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:40.405 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:40.405 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:40.405 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:40.405 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:40.405 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:40.405 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:40.406 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:40.424 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:40.424 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:40.425 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:40.425 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:40.425 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:40.425 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:40.425 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:40.426 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:40.426 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:40.444 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:40.444 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:40.445 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:40.445 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:40.445 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:40.445 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:40.445 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:40.445 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:40.446 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:40.464 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:40.464 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:40.465 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:40.465 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:40.465 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:40.465 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:40.466 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:40.466 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:40.466 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:40.484 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:40.484 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:40.485 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:40.485 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:40.485 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:40.485 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:40.485 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:40.485 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:40.486 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:40.524 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:40.524 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:40.525 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:40.525 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:40.525 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:40.525 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:40.525 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:40.525 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:40.526 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:40.544 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:40.544 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:40.545 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:40.545 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:40.545 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:40.545 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:40.545 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:40.545 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:40.546 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:40.547 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=162), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [59ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [79ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [99ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [141ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [141ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [142ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [161ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [161ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [162ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, deferred=Deferred@247748359(state=PENDING, result=null, callback=(continuation of Deferred@473443654 after retry RPC after error@1030816798) -> (continuation of Deferred@1200167617 after retry RPC after error@1800999256) -> (continuation of Deferred@920015720 after retry RPC after error@53665880) -> (continuation of Deferred@216182253 after retry RPC after error@798146296) -> (continuation of Deferred@735831409 after retry RPC after error@1065349234) -> (continuation of Deferred@644591541 after retry RPC after error@481603838) -> (continuation of Deferred@1756380788 after retry RPC after error@671919984), errback=(continuation of Deferred@473443654 after retry RPC after error@1030816798) -> (continuation of Deferred@1200167617 after retry RPC after error@1800999256) -> (continuation of Deferred@920015720 after retry RPC after error@53665880) -> (continuation of Deferred@216182253 after retry RPC after error@798146296) -> (continuation of Deferred@735831409 after retry RPC after error@1065349234) -> (continuation of Deferred@644591541 after retry RPC after error@481603838) -> (continuation of Deferred@1756380788 after retry RPC after error@671919984)))
21:04:40.548 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=162), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [59ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [79ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [99ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [141ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [141ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [142ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [161ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [161ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [162ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, deferred=Deferred@247748359(state=PENDING, result=null, callback=(continuation of Deferred@473443654 after retry RPC after error@1030816798) -> (continuation of Deferred@1200167617 after retry RPC after error@1800999256) -> (continuation of Deferred@920015720 after retry RPC after error@53665880) -> (continuation of Deferred@216182253 after retry RPC after error@798146296) -> (continuation of Deferred@735831409 after retry RPC after error@1065349234) -> (continuation of Deferred@644591541 after retry RPC after error@481603838) -> (continuation of Deferred@1756380788 after retry RPC after error@671919984), errback=(continuation of Deferred@473443654 after retry RPC after error@1030816798) -> (continuation of Deferred@1200167617 after retry RPC after error@1800999256) -> (continuation of Deferred@920015720 after retry RPC after error@53665880) -> (continuation of Deferred@216182253 after retry RPC after error@798146296) -> (continuation of Deferred@735831409 after retry RPC after error@1065349234) -> (continuation of Deferred@644591541 after retry RPC after error@481603838) -> (continuation of Deferred@1756380788 after retry RPC after error@671919984)))
21:04:40.564 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:40.564 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:40.565 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:40.565 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:40.565 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:40.565 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:40.565 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:40.566 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:40.567 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:40.584 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:40.585 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:40.585 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:40.585 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:40.585 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:40.585 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:40.586 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:40.586 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:40.586 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:40.604 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:40.605 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:40.605 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:40.605 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:40.605 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:40.605 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:40.606 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:40.606 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:40.606 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:40.624 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:40.624 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:40.625 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:40.625 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:40.625 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:40.625 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:40.625 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:40.625 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:40.626 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:40.644 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:40.644 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:40.645 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:40.645 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:40.645 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:40.645 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:40.645 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:40.645 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:40.646 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:40.664 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:40.664 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:40.665 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:40.665 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:40.665 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:40.665 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:40.665 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:40.666 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:40.666 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:40.724 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:40.725 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:40.725 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:40.725 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:40.725 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:40.725 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:40.725 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:40.726 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:40.726 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:40.727 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=162), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [3ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [161ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [162ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [162ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, deferred=Deferred@255734024(state=PENDING, result=null, callback=(continuation of Deferred@1358274009 after retry RPC after error@1580242649) -> (continuation of Deferred@1218337731 after retry RPC after error@920028585) -> (continuation of Deferred@1244777983 after retry RPC after error@283970622) -> (continuation of Deferred@714546595 after retry RPC after error@686702229) -> (continuation of Deferred@1827950658 after retry RPC after error@343944478) -> (continuation of Deferred@115884072 after retry RPC after error@1605098742), errback=(continuation of Deferred@1358274009 after retry RPC after error@1580242649) -> (continuation of Deferred@1218337731 after retry RPC after error@920028585) -> (continuation of Deferred@1244777983 after retry RPC after error@283970622) -> (continuation of Deferred@714546595 after retry RPC after error@686702229) -> (continuation of Deferred@1827950658 after retry RPC after error@343944478) -> (continuation of Deferred@115884072 after retry RPC after error@1605098742)))
21:04:40.727 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=162), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [3ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [161ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [162ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [162ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, deferred=Deferred@255734024(state=PENDING, result=null, callback=(continuation of Deferred@1358274009 after retry RPC after error@1580242649) -> (continuation of Deferred@1218337731 after retry RPC after error@920028585) -> (continuation of Deferred@1244777983 after retry RPC after error@283970622) -> (continuation of Deferred@714546595 after retry RPC after error@686702229) -> (continuation of Deferred@1827950658 after retry RPC after error@343944478) -> (continuation of Deferred@115884072 after retry RPC after error@1605098742), errback=(continuation of Deferred@1358274009 after retry RPC after error@1580242649) -> (continuation of Deferred@1218337731 after retry RPC after error@920028585) -> (continuation of Deferred@1244777983 after retry RPC after error@283970622) -> (continuation of Deferred@714546595 after retry RPC after error@686702229) -> (continuation of Deferred@1827950658 after retry RPC after error@343944478) -> (continuation of Deferred@115884072 after retry RPC after error@1605098742)))
21:04:40.744 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:40.744 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:40.745 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:40.745 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:40.745 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:40.745 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:40.745 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:40.745 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:40.746 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:40.764 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:40.764 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:40.765 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:40.765 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:40.765 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:40.765 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:40.765 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:40.765 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:40.766 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:40.784 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:40.785 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:40.785 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:40.785 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:40.785 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:40.785 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:40.785 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:40.785 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:40.786 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:40.804 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:40.805 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:40.805 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:40.805 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:40.805 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:40.805 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:40.805 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:40.805 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:40.807 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:40.824 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:40.825 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:40.825 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:40.825 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:40.825 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:40.825 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:40.825 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:40.825 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:40.826 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:40.844 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:40.845 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:40.845 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:40.845 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:40.845 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:40.845 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:40.845 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:40.845 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:40.846 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:40.884 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:40.885 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:40.885 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:40.885 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:40.885 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:40.886 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:40.886 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:40.886 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:40.887 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:40.924 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:40.924 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:40.925 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:40.925 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:40.925 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:40.925 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:40.925 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:40.925 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:40.926 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:40.926 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=181), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [179ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [181ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [181ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [182ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, deferred=Deferred@272772934(state=PENDING, result=null, callback=(continuation of Deferred@1903538168 after retry RPC after error@1533464853) -> (continuation of Deferred@584283051 after retry RPC after error@594165928) -> (continuation of Deferred@2143035730 after retry RPC after error@314550729) -> (continuation of Deferred@734580254 after retry RPC after error@1524016335) -> (continuation of Deferred@1149151350 after retry RPC after error@727757009) -> (continuation of Deferred@1186276253 after retry RPC after error@699012952), errback=(continuation of Deferred@1903538168 after retry RPC after error@1533464853) -> (continuation of Deferred@584283051 after retry RPC after error@594165928) -> (continuation of Deferred@2143035730 after retry RPC after error@314550729) -> (continuation of Deferred@734580254 after retry RPC after error@1524016335) -> (continuation of Deferred@1149151350 after retry RPC after error@727757009) -> (continuation of Deferred@1186276253 after retry RPC after error@699012952)))
21:04:40.927 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=181), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [179ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [181ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [181ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [182ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, deferred=Deferred@272772934(state=PENDING, result=null, callback=(continuation of Deferred@1903538168 after retry RPC after error@1533464853) -> (continuation of Deferred@584283051 after retry RPC after error@594165928) -> (continuation of Deferred@2143035730 after retry RPC after error@314550729) -> (continuation of Deferred@734580254 after retry RPC after error@1524016335) -> (continuation of Deferred@1149151350 after retry RPC after error@727757009) -> (continuation of Deferred@1186276253 after retry RPC after error@699012952), errback=(continuation of Deferred@1903538168 after retry RPC after error@1533464853) -> (continuation of Deferred@584283051 after retry RPC after error@594165928) -> (continuation of Deferred@2143035730 after retry RPC after error@314550729) -> (continuation of Deferred@734580254 after retry RPC after error@1524016335) -> (continuation of Deferred@1149151350 after retry RPC after error@727757009) -> (continuation of Deferred@1186276253 after retry RPC after error@699012952)))
21:04:40.945 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:40.945 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:40.946 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:40.946 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:40.946 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:40.946 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:40.946 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:40.946 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:40.947 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:40.965 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:40.965 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:40.966 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:40.966 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:40.966 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:40.966 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:40.966 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:40.966 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:40.967 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:40.984 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:40.985 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:40.985 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:40.986 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:40.986 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:40.986 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:40.986 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:40.986 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:40.987 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:41.004 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:41.005 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:41.005 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:41.005 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:41.005 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:41.005 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:41.005 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:41.005 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:41.006 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:41.024 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:41.024 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:41.025 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:41.025 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:41.025 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:41.025 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:41.025 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:41.025 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:41.026 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:41.064 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:41.064 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:41.065 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:41.065 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:41.065 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:41.065 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:41.065 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:41.065 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:41.066 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:41.124 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:41.125 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:41.125 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:41.125 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:41.125 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:41.125 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:41.125 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:41.125 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:41.126 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:41.127 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=182), Traces: [0ms] refreshing cache from master, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [3ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [23ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [181ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [181ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [182ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [183ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, deferred=Deferred@1162266211(state=PENDING, result=null, callback=(continuation of Deferred@925995016 after retry RPC after error@1727766618) -> (continuation of Deferred@910696386 after retry RPC after error@1162785462) -> (continuation of Deferred@2118441927 after retry RPC after error@1166429477) -> (continuation of Deferred@860286040 after retry RPC after error@61678468) -> (continuation of Deferred@1542408917 after retry RPC after error@754437096) -> (continuation of Deferred@676845058 after retry RPC after error@1108107663), errback=(continuation of Deferred@925995016 after retry RPC after error@1727766618) -> (continuation of Deferred@910696386 after retry RPC after error@1162785462) -> (continuation of Deferred@2118441927 after retry RPC after error@1166429477) -> (continuation of Deferred@860286040 after retry RPC after error@61678468) -> (continuation of Deferred@1542408917 after retry RPC after error@754437096) -> (continuation of Deferred@676845058 after retry RPC after error@1108107663)))
21:04:41.127 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=182), Traces: [0ms] refreshing cache from master, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [3ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [23ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [181ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [181ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [182ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [183ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, deferred=Deferred@1162266211(state=PENDING, result=null, callback=(continuation of Deferred@925995016 after retry RPC after error@1727766618) -> (continuation of Deferred@910696386 after retry RPC after error@1162785462) -> (continuation of Deferred@2118441927 after retry RPC after error@1166429477) -> (continuation of Deferred@860286040 after retry RPC after error@61678468) -> (continuation of Deferred@1542408917 after retry RPC after error@754437096) -> (continuation of Deferred@676845058 after retry RPC after error@1108107663), errback=(continuation of Deferred@925995016 after retry RPC after error@1727766618) -> (continuation of Deferred@910696386 after retry RPC after error@1162785462) -> (continuation of Deferred@2118441927 after retry RPC after error@1166429477) -> (continuation of Deferred@860286040 after retry RPC after error@61678468) -> (continuation of Deferred@1542408917 after retry RPC after error@754437096) -> (continuation of Deferred@676845058 after retry RPC after error@1108107663)))
21:04:41.144 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:41.145 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:41.145 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:41.145 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:41.145 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:41.145 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:41.145 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:41.145 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:41.147 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:41.164 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:41.165 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:41.165 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:41.165 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:41.165 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:41.165 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:41.165 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:41.165 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:41.167 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:41.184 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:41.185 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:41.185 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:41.185 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:41.185 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:41.185 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:41.185 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:41.185 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:41.186 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:41.204 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:41.205 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:41.205 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:41.205 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:41.205 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:41.205 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:41.205 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:41.205 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:41.206 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:41.224 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:41.225 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:41.225 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:41.225 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:41.225 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:41.225 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:41.225 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:41.226 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:41.227 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:41.264 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:41.265 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:41.265 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:41.265 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:41.266 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:41.266 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:41.266 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:41.266 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:41.267 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:41.324 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:41.324 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:41.325 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:41.325 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:41.325 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:41.325 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:41.325 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:41.325 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:41.326 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:41.327 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=182), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [123ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [181ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [181ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [182ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, deferred=Deferred@860028168(state=PENDING, result=null, callback=(continuation of Deferred@52928702 after retry RPC after error@77228439) -> (continuation of Deferred@1779213383 after retry RPC after error@1317187251) -> (continuation of Deferred@1881064845 after retry RPC after error@664784072) -> (continuation of Deferred@675183328 after retry RPC after error@1930604130) -> (continuation of Deferred@1973829408 after retry RPC after error@747001040) -> (continuation of Deferred@1999946458 after retry RPC after error@296948704), errback=(continuation of Deferred@52928702 after retry RPC after error@77228439) -> (continuation of Deferred@1779213383 after retry RPC after error@1317187251) -> (continuation of Deferred@1881064845 after retry RPC after error@664784072) -> (continuation of Deferred@675183328 after retry RPC after error@1930604130) -> (continuation of Deferred@1973829408 after retry RPC after error@747001040) -> (continuation of Deferred@1999946458 after retry RPC after error@296948704)))
21:04:41.327 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=182), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [123ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [181ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [181ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [182ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, deferred=Deferred@860028168(state=PENDING, result=null, callback=(continuation of Deferred@52928702 after retry RPC after error@77228439) -> (continuation of Deferred@1779213383 after retry RPC after error@1317187251) -> (continuation of Deferred@1881064845 after retry RPC after error@664784072) -> (continuation of Deferred@675183328 after retry RPC after error@1930604130) -> (continuation of Deferred@1973829408 after retry RPC after error@747001040) -> (continuation of Deferred@1999946458 after retry RPC after error@296948704), errback=(continuation of Deferred@52928702 after retry RPC after error@77228439) -> (continuation of Deferred@1779213383 after retry RPC after error@1317187251) -> (continuation of Deferred@1881064845 after retry RPC after error@664784072) -> (continuation of Deferred@675183328 after retry RPC after error@1930604130) -> (continuation of Deferred@1973829408 after retry RPC after error@747001040) -> (continuation of Deferred@1999946458 after retry RPC after error@296948704)))
21:04:41.344 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:41.344 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:41.345 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:41.345 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:41.345 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:41.345 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:41.345 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:41.345 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:41.346 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:41.364 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:41.364 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:41.365 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:41.365 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:41.365 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:41.365 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:41.365 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:41.365 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:41.366 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:41.384 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:41.384 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:41.385 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:41.385 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:41.385 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:41.385 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:41.385 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:41.385 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:41.386 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:41.404 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:41.404 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:41.405 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:41.405 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:41.405 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:41.405 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:41.405 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:41.405 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:41.406 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:41.424 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:41.424 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:41.425 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:41.425 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:41.425 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:41.425 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:41.425 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:41.425 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:41.426 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:41.444 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:41.444 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:41.445 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:41.445 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:41.445 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:41.445 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:41.445 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:41.445 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:41.446 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:41.484 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:41.484 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:41.485 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:41.485 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:41.485 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:41.485 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:41.485 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:41.485 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:41.486 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:41.584 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:41.585 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:41.585 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:41.585 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:41.585 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:41.585 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:41.585 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:41.585 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:41.587 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:41.587 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=243), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [79ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [99ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [141ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [141ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [142ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [241ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [241ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [242ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [243ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, deferred=Deferred@1882180224(state=PENDING, result=null, callback=(continuation of Deferred@711433406 after retry RPC after error@490118840) -> (continuation of Deferred@1441379113 after retry RPC after error@1528568030) -> (continuation of Deferred@1300823407 after retry RPC after error@1684171064) -> (continuation of Deferred@2089479782 after retry RPC after error@1206633409) -> (continuation of Deferred@1300377046 after retry RPC after error@987670263) -> (continuation of Deferred@294422036 after retry RPC after error@140552751) -> (continuation of Deferred@1422969448 after retry RPC after error@469116699), errback=(continuation of Deferred@711433406 after retry RPC after error@490118840) -> (continuation of Deferred@1441379113 after retry RPC after error@1528568030) -> (continuation of Deferred@1300823407 after retry RPC after error@1684171064) -> (continuation of Deferred@2089479782 after retry RPC after error@1206633409) -> (continuation of Deferred@1300377046 after retry RPC after error@987670263) -> (continuation of Deferred@294422036 after retry RPC after error@140552751) -> (continuation of Deferred@1422969448 after retry RPC after error@469116699)))
21:04:41.588 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=243), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [79ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [99ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [141ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [141ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [142ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [241ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [241ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [242ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [243ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, deferred=Deferred@1882180224(state=PENDING, result=null, callback=(continuation of Deferred@711433406 after retry RPC after error@490118840) -> (continuation of Deferred@1441379113 after retry RPC after error@1528568030) -> (continuation of Deferred@1300823407 after retry RPC after error@1684171064) -> (continuation of Deferred@2089479782 after retry RPC after error@1206633409) -> (continuation of Deferred@1300377046 after retry RPC after error@987670263) -> (continuation of Deferred@294422036 after retry RPC after error@140552751) -> (continuation of Deferred@1422969448 after retry RPC after error@469116699), errback=(continuation of Deferred@711433406 after retry RPC after error@490118840) -> (continuation of Deferred@1441379113 after retry RPC after error@1528568030) -> (continuation of Deferred@1300823407 after retry RPC after error@1684171064) -> (continuation of Deferred@2089479782 after retry RPC after error@1206633409) -> (continuation of Deferred@1300377046 after retry RPC after error@987670263) -> (continuation of Deferred@294422036 after retry RPC after error@140552751) -> (continuation of Deferred@1422969448 after retry RPC after error@469116699)))
21:04:41.604 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:41.605 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:41.605 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:41.605 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:41.605 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:41.605 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:41.605 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:41.605 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:41.606 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:41.624 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:41.624 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:41.625 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:41.625 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:41.625 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:41.625 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:41.625 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:41.626 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:41.626 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:41.644 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:41.644 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:41.645 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:41.645 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:41.645 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:41.645 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:41.645 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:41.645 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:41.646 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:41.664 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:41.664 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:41.665 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:41.665 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:41.665 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:41.665 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:41.665 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:41.665 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:41.666 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:41.684 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:41.684 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:41.685 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:41.685 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:41.685 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:41.685 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:41.685 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:41.685 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:41.686 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:41.704 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:41.704 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:41.705 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:41.705 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:41.705 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:41.705 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:41.705 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:41.705 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:41.706 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:41.724 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:41.724 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:41.725 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:41.725 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:41.725 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:41.725 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:41.725 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:41.725 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:41.726 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:41.844 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:41.844 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:41.845 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:41.845 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:41.845 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:41.845 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:41.845 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:41.845 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:41.846 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:41.847 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=242), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [239ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [241ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [241ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [242ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [242ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, deferred=Deferred@1889442482(state=PENDING, result=null, callback=(continuation of Deferred@1064349645 after retry RPC after error@518600788) -> (continuation of Deferred@582325810 after retry RPC after error@645579740) -> (continuation of Deferred@483266073 after retry RPC after error@1261612620) -> (continuation of Deferred@1161340190 after retry RPC after error@900524614) -> (continuation of Deferred@1509852204 after retry RPC after error@123732369) -> (continuation of Deferred@133190710 after retry RPC after error@1440510036) -> (continuation of Deferred@2024689959 after retry RPC after error@2135276330), errback=(continuation of Deferred@1064349645 after retry RPC after error@518600788) -> (continuation of Deferred@582325810 after retry RPC after error@645579740) -> (continuation of Deferred@483266073 after retry RPC after error@1261612620) -> (continuation of Deferred@1161340190 after retry RPC after error@900524614) -> (continuation of Deferred@1509852204 after retry RPC after error@123732369) -> (continuation of Deferred@133190710 after retry RPC after error@1440510036) -> (continuation of Deferred@2024689959 after retry RPC after error@2135276330)))
21:04:41.847 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=242), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [239ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [241ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [241ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [242ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [242ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, deferred=Deferred@1889442482(state=PENDING, result=null, callback=(continuation of Deferred@1064349645 after retry RPC after error@518600788) -> (continuation of Deferred@582325810 after retry RPC after error@645579740) -> (continuation of Deferred@483266073 after retry RPC after error@1261612620) -> (continuation of Deferred@1161340190 after retry RPC after error@900524614) -> (continuation of Deferred@1509852204 after retry RPC after error@123732369) -> (continuation of Deferred@133190710 after retry RPC after error@1440510036) -> (continuation of Deferred@2024689959 after retry RPC after error@2135276330), errback=(continuation of Deferred@1064349645 after retry RPC after error@518600788) -> (continuation of Deferred@582325810 after retry RPC after error@645579740) -> (continuation of Deferred@483266073 after retry RPC after error@1261612620) -> (continuation of Deferred@1161340190 after retry RPC after error@900524614) -> (continuation of Deferred@1509852204 after retry RPC after error@123732369) -> (continuation of Deferred@133190710 after retry RPC after error@1440510036) -> (continuation of Deferred@2024689959 after retry RPC after error@2135276330)))
21:04:41.864 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:41.864 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:41.864 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:41.865 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:41.865 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:41.865 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:41.865 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:41.865 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:41.866 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:41.884 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:41.884 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:41.885 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:41.885 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:41.885 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:41.885 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:41.885 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:41.885 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:41.886 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:41.905 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:41.905 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:41.905 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:41.906 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:41.906 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:41.906 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:41.906 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:41.906 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:41.907 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:41.925 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:41.925 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:41.926 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:41.926 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:41.926 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:41.926 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:41.926 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:41.926 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:41.927 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:41.944 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:41.944 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:41.945 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:41.945 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:41.945 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:41.945 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:41.945 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:41.945 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:41.946 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:41.964 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:41.964 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:41.965 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:41.965 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:41.965 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:41.965 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:41.965 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:41.965 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:41.966 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:42.004 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:42.004 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:42.005 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:42.005 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:42.005 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:42.005 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:42.005 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:42.005 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:42.006 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:42.084 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:42.085 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:42.085 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:42.085 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:42.085 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:42.085 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:42.086 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:42.086 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:42.086 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:42.087 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=222), Traces: [0ms] refreshing cache from master, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [23ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [41ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [42ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [42ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [43ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [43ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [44ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [44ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [61ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [62ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [62ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [63ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [63ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [64ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [64ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [80ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [83ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [100ms] refreshing cache from master, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [103ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [141ms] refreshing cache from master, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [142ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [142ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [143ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [143ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [221ms] refreshing cache from master, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [222ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [222ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [222ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [223ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [223ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [223ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, deferred=Deferred@1750671620(state=PENDING, result=null, callback=(continuation of Deferred@620005058 after retry RPC after error@1164412307) -> (continuation of Deferred@945132134 after retry RPC after error@430298473) -> (continuation of Deferred@434602505 after retry RPC after error@1624623035) -> (continuation of Deferred@1946664805 after retry RPC after error@1128149521) -> (continuation of Deferred@1718359989 after retry RPC after error@684387548) -> (continuation of Deferred@1644134967 after retry RPC after error@1399032483) -> (continuation of Deferred@1437925061 after retry RPC after error@711410851), errback=(continuation of Deferred@620005058 after retry RPC after error@1164412307) -> (continuation of Deferred@945132134 after retry RPC after error@430298473) -> (continuation of Deferred@434602505 after retry RPC after error@1624623035) -> (continuation of Deferred@1946664805 after retry RPC after error@1128149521) -> (continuation of Deferred@1718359989 after retry RPC after error@684387548) -> (continuation of Deferred@1644134967 after retry RPC after error@1399032483) -> (continuation of Deferred@1437925061 after retry RPC after error@711410851)))
21:04:42.087 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=222), Traces: [0ms] refreshing cache from master, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [23ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [41ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [42ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [42ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [43ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [43ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [44ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [44ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [61ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [62ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [62ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [63ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [63ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [64ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [64ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [80ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [83ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [100ms] refreshing cache from master, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [103ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [141ms] refreshing cache from master, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [142ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [142ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [143ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [143ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [221ms] refreshing cache from master, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [222ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [222ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [222ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [223ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [223ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [223ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, deferred=Deferred@1750671620(state=PENDING, result=null, callback=(continuation of Deferred@620005058 after retry RPC after error@1164412307) -> (continuation of Deferred@945132134 after retry RPC after error@430298473) -> (continuation of Deferred@434602505 after retry RPC after error@1624623035) -> (continuation of Deferred@1946664805 after retry RPC after error@1128149521) -> (continuation of Deferred@1718359989 after retry RPC after error@684387548) -> (continuation of Deferred@1644134967 after retry RPC after error@1399032483) -> (continuation of Deferred@1437925061 after retry RPC after error@711410851), errback=(continuation of Deferred@620005058 after retry RPC after error@1164412307) -> (continuation of Deferred@945132134 after retry RPC after error@430298473) -> (continuation of Deferred@434602505 after retry RPC after error@1624623035) -> (continuation of Deferred@1946664805 after retry RPC after error@1128149521) -> (continuation of Deferred@1718359989 after retry RPC after error@684387548) -> (continuation of Deferred@1644134967 after retry RPC after error@1399032483) -> (continuation of Deferred@1437925061 after retry RPC after error@711410851)))
21:04:42.104 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:42.105 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:42.105 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:42.105 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:42.105 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:42.105 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:42.105 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:42.105 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:42.106 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:42.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:42.123492 24405 raft_consensus.cc:493] T 00000000000000000000000000000000 P eb664add87f44013b596f4cc01c9a41c [term 4 FOLLOWER]: Starting pre-election (detected failure of leader c520077acf3341e29447c6c2b8d3f74c)
21:04:42.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:42.123723 24405 raft_consensus.cc:515] T 00000000000000000000000000000000 P eb664add87f44013b596f4cc01c9a41c [term 4 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e9dd218ca2244f5a934aac2b7e58eb1d" member_type: VOTER last_known_addr { host: "127.22.168.62" port: 37239 } } peers { permanent_uuid: "c520077acf3341e29447c6c2b8d3f74c" member_type: VOTER last_known_addr { host: "127.22.168.61" port: 32947 } } peers { permanent_uuid: "eb664add87f44013b596f4cc01c9a41c" member_type: VOTER last_known_addr { host: "127.22.168.60" port: 33247 } }
21:04:42.124 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:42.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:42.124616 24405 leader_election.cc:290] T 00000000000000000000000000000000 P eb664add87f44013b596f4cc01c9a41c [CANDIDATE]: Term 5 pre-election: Requested pre-vote from peers e9dd218ca2244f5a934aac2b7e58eb1d (127.22.168.62:37239), c520077acf3341e29447c6c2b8d3f74c (127.22.168.61:32947)
21:04:42.125 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:42.125 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:42.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:42.125797 24086 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.22.168.62:37239: connect: Connection refused (error 111) [suppressed 3 similar messages]
21:04:42.126 [INFO - pool-40-thread-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:42.126 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:42.126 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:42.126 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:42.127 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:42.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:42.127374 24086 leader_election.cc:336] T 00000000000000000000000000000000 P eb664add87f44013b596f4cc01c9a41c [CANDIDATE]: Term 5 pre-election: RPC error from VoteRequest() call to peer e9dd218ca2244f5a934aac2b7e58eb1d (127.22.168.62:37239): Network error: Client connection negotiation failed: client connection to 127.22.168.62:37239: connect: Connection refused (error 111)
21:04:42.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:42.127697 24089 leader_election.cc:336] T 00000000000000000000000000000000 P eb664add87f44013b596f4cc01c9a41c [CANDIDATE]: Term 5 pre-election: RPC error from VoteRequest() call to peer c520077acf3341e29447c6c2b8d3f74c (127.22.168.61:32947): Network error: Client connection negotiation failed: client connection to 127.22.168.61:32947: connect: Connection refused (error 111)
21:04:42.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:42.127858 24089 leader_election.cc:304] T 00000000000000000000000000000000 P eb664add87f44013b596f4cc01c9a41c [CANDIDATE]: Term 5 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: eb664add87f44013b596f4cc01c9a41c; no voters: c520077acf3341e29447c6c2b8d3f74c, e9dd218ca2244f5a934aac2b7e58eb1d
21:04:42.128 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:42.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:42.128154 24405 raft_consensus.cc:2749] T 00000000000000000000000000000000 P eb664add87f44013b596f4cc01c9a41c [term 4 FOLLOWER]: Leader pre-election lost for term 5. Reason: could not achieve majority
21:04:42.144 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:42.145 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:42.145 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:42.145 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:42.145 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:42.145 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:42.145 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:42.146 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:42.147 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:42.164 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:42.165 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:42.165 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:42.165 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:42.165 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:42.165 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:42.166 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:42.166 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:42.167 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:42.184 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:42.185 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:42.185 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:42.185 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:42.185 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:42.185 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:42.185 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:42.185 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:42.187 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:42.204 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:42.205 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:42.205 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:42.205 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:42.205 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:42.205 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:42.205 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:42.205 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:42.206 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:42.244 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:42.245 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:42.245 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:42.245 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:42.245 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:42.245 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:42.245 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:42.246 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:42.247 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:42.304 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:42.305 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:42.305 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:42.305 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:42.305 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:42.306 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:42.306 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:42.306 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:42.307 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:42.307 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=202), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [22ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [22ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [23ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [24ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [141ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [141ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [142ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [143ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [202ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [202ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [202ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [203ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, deferred=Deferred@974391515(state=PENDING, result=null, callback=(continuation of Deferred@1868789339 after retry RPC after error@2116570835) -> (continuation of Deferred@831475635 after retry RPC after error@734514057) -> (continuation of Deferred@1600303364 after retry RPC after error@1669346029) -> (continuation of Deferred@674827257 after retry RPC after error@1035405044) -> (continuation of Deferred@1998023383 after retry RPC after error@367147394) -> (continuation of Deferred@1977916932 after retry RPC after error@1885452010) -> (continuation of Deferred@1013722083 after retry RPC after error@944122821), errback=(continuation of Deferred@1868789339 after retry RPC after error@2116570835) -> (continuation of Deferred@831475635 after retry RPC after error@734514057) -> (continuation of Deferred@1600303364 after retry RPC after error@1669346029) -> (continuation of Deferred@674827257 after retry RPC after error@1035405044) -> (continuation of Deferred@1998023383 after retry RPC after error@367147394) -> (continuation of Deferred@1977916932 after retry RPC after error@1885452010) -> (continuation of Deferred@1013722083 after retry RPC after error@944122821)))
21:04:42.308 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=202), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [22ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [22ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [23ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [24ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [141ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [141ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [142ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [143ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [202ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [202ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [202ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [203ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, deferred=Deferred@974391515(state=PENDING, result=null, callback=(continuation of Deferred@1868789339 after retry RPC after error@2116570835) -> (continuation of Deferred@831475635 after retry RPC after error@734514057) -> (continuation of Deferred@1600303364 after retry RPC after error@1669346029) -> (continuation of Deferred@674827257 after retry RPC after error@1035405044) -> (continuation of Deferred@1998023383 after retry RPC after error@367147394) -> (continuation of Deferred@1977916932 after retry RPC after error@1885452010) -> (continuation of Deferred@1013722083 after retry RPC after error@944122821), errback=(continuation of Deferred@1868789339 after retry RPC after error@2116570835) -> (continuation of Deferred@831475635 after retry RPC after error@734514057) -> (continuation of Deferred@1600303364 after retry RPC after error@1669346029) -> (continuation of Deferred@674827257 after retry RPC after error@1035405044) -> (continuation of Deferred@1998023383 after retry RPC after error@367147394) -> (continuation of Deferred@1977916932 after retry RPC after error@1885452010) -> (continuation of Deferred@1013722083 after retry RPC after error@944122821)))
21:04:42.324 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:42.325 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:42.325 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:42.325 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:42.325 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:42.325 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:42.326 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:42.326 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:42.326 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:42.344 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:42.345 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:42.345 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:42.345 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:42.345 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:42.347 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:42.347 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:42.348 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:42.348 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:42.364 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:42.364 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:42.365 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:42.365 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:42.365 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:42.365 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:42.365 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:42.365 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:42.366 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:42.384 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:42.384 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:42.385 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:42.385 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:42.385 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:42.385 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:42.385 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:42.385 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:42.386 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:42.404 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:42.404 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:42.405 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:42.405 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:42.405 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:42.405 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:42.405 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:42.405 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:42.406 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:42.444 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:42.444 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:42.445 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:42.445 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:42.445 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:42.445 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:42.446 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:42.446 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:42.446 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:42.464 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:42.464 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:42.465 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:42.465 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:42.465 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:42.465 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:42.465 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:42.465 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:42.466 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:42.504 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:42.505 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:42.505 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:42.505 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:42.505 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:42.505 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:42.506 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:42.506 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:42.507 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:42.508 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=182), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [23ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [23ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [24ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [141ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [141ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [142ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [181ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [182ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [183ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [183ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, deferred=Deferred@1685160029(state=PENDING, result=null, callback=(continuation of Deferred@1548315452 after retry RPC after error@25684124) -> (continuation of Deferred@156612342 after retry RPC after error@1640938729) -> (continuation of Deferred@1984506727 after retry RPC after error@1219821114) -> (continuation of Deferred@1819411420 after retry RPC after error@2018493263) -> (continuation of Deferred@1206050098 after retry RPC after error@684244698) -> (continuation of Deferred@1423455484 after retry RPC after error@1376506003) -> (continuation of Deferred@55676077 after retry RPC after error@1845677171), errback=(continuation of Deferred@1548315452 after retry RPC after error@25684124) -> (continuation of Deferred@156612342 after retry RPC after error@1640938729) -> (continuation of Deferred@1984506727 after retry RPC after error@1219821114) -> (continuation of Deferred@1819411420 after retry RPC after error@2018493263) -> (continuation of Deferred@1206050098 after retry RPC after error@684244698) -> (continuation of Deferred@1423455484 after retry RPC after error@1376506003) -> (continuation of Deferred@55676077 after retry RPC after error@1845677171)))
21:04:42.508 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=182), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [23ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [23ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [24ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [141ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [141ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [142ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [181ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [182ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [183ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [183ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, deferred=Deferred@1685160029(state=PENDING, result=null, callback=(continuation of Deferred@1548315452 after retry RPC after error@25684124) -> (continuation of Deferred@156612342 after retry RPC after error@1640938729) -> (continuation of Deferred@1984506727 after retry RPC after error@1219821114) -> (continuation of Deferred@1819411420 after retry RPC after error@2018493263) -> (continuation of Deferred@1206050098 after retry RPC after error@684244698) -> (continuation of Deferred@1423455484 after retry RPC after error@1376506003) -> (continuation of Deferred@55676077 after retry RPC after error@1845677171), errback=(continuation of Deferred@1548315452 after retry RPC after error@25684124) -> (continuation of Deferred@156612342 after retry RPC after error@1640938729) -> (continuation of Deferred@1984506727 after retry RPC after error@1219821114) -> (continuation of Deferred@1819411420 after retry RPC after error@2018493263) -> (continuation of Deferred@1206050098 after retry RPC after error@684244698) -> (continuation of Deferred@1423455484 after retry RPC after error@1376506003) -> (continuation of Deferred@55676077 after retry RPC after error@1845677171)))
21:04:42.524 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:42.525 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:42.525 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:42.525 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:42.525 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:42.525 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:42.525 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:42.525 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:42.526 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:42.544 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:42.544 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:42.545 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:42.545 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:42.545 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:42.546 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:42.546 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:42.546 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:42.547 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:42.564 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:42.564 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:42.565 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:42.565 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:42.565 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:42.565 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:42.565 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:42.565 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:42.566 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:42.584 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:42.584 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:42.585 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:42.585 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:42.585 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:42.585 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:42.585 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:42.585 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:42.586 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:42.604 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:42.604 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:42.605 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:42.605 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:42.605 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:42.606 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:42.606 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:42.606 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:42.607 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:42.644 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:42.644 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:42.645 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:42.645 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:42.645 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:42.646 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:42.646 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:42.646 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:42.647 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:42.704 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:42.705 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:42.706 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:42.706 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:42.706 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:42.706 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:42.706 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:42.707 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:42.707 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:42.765 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:42.765 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:42.765 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:42.765 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:42.766 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:42.766 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:42.766 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:42.766 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:42.767 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:42.767 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=242), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [23ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [123ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [182ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [182ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [182ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [183ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [242ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [242ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [242ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [243ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, deferred=Deferred@1110623808(state=PENDING, result=null, callback=(continuation of Deferred@298718724 after retry RPC after error@964344226) -> (continuation of Deferred@1772852085 after retry RPC after error@1031755974) -> (continuation of Deferred@1549797211 after retry RPC after error@817562332) -> (continuation of Deferred@96345989 after retry RPC after error@1111547548) -> (continuation of Deferred@516309840 after retry RPC after error@1979912118) -> (continuation of Deferred@53769412 after retry RPC after error@1195703808) -> (continuation of Deferred@1807022459 after retry RPC after error@94643460), errback=(continuation of Deferred@298718724 after retry RPC after error@964344226) -> (continuation of Deferred@1772852085 after retry RPC after error@1031755974) -> (continuation of Deferred@1549797211 after retry RPC after error@817562332) -> (continuation of Deferred@96345989 after retry RPC after error@1111547548) -> (continuation of Deferred@516309840 after retry RPC after error@1979912118) -> (continuation of Deferred@53769412 after retry RPC after error@1195703808) -> (continuation of Deferred@1807022459 after retry RPC after error@94643460)))
21:04:42.767 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=242), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [23ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [123ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [182ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [182ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [182ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [183ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [242ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [242ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [242ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [243ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, deferred=Deferred@1110623808(state=PENDING, result=null, callback=(continuation of Deferred@298718724 after retry RPC after error@964344226) -> (continuation of Deferred@1772852085 after retry RPC after error@1031755974) -> (continuation of Deferred@1549797211 after retry RPC after error@817562332) -> (continuation of Deferred@96345989 after retry RPC after error@1111547548) -> (continuation of Deferred@516309840 after retry RPC after error@1979912118) -> (continuation of Deferred@53769412 after retry RPC after error@1195703808) -> (continuation of Deferred@1807022459 after retry RPC after error@94643460), errback=(continuation of Deferred@298718724 after retry RPC after error@964344226) -> (continuation of Deferred@1772852085 after retry RPC after error@1031755974) -> (continuation of Deferred@1549797211 after retry RPC after error@817562332) -> (continuation of Deferred@96345989 after retry RPC after error@1111547548) -> (continuation of Deferred@516309840 after retry RPC after error@1979912118) -> (continuation of Deferred@53769412 after retry RPC after error@1195703808) -> (continuation of Deferred@1807022459 after retry RPC after error@94643460)))
21:04:42.784 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:42.785 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:42.785 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:42.785 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:42.786 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:42.786 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:42.786 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:42.786 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:42.787 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:42.804 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:42.805 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:42.805 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:42.805 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:42.805 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:42.805 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:42.806 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:42.806 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:42.806 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:42.824 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:42.825 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:42.825 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:42.825 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:42.825 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:42.826 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:42.826 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:42.826 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:42.827 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:42.844 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:42.845 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:42.846 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:42.846 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:42.846 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:42.846 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:42.846 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:42.846 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:42.847 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:42.864 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:42.864 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:42.865 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:42.865 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:42.865 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:42.865 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:42.865 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:42.865 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:42.866 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:42.884 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:42.884 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:42.885 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:42.885 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:42.885 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:42.885 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:42.885 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:42.885 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:42.886 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:42.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:42.903659 23569 master_proxy_rpc.cc:177] Failed to determine new leader Master: Timed out: ConnectToClusterRpc(addrs: 127.22.168.62:37239,127.22.168.61:32947,127.22.168.60:33247, num_attempts: 96) passed its deadline: Not found: no leader found: ConnectToClusterRpc(addrs: 127.22.168.62:37239,127.22.168.61:32947,127.22.168.60:33247, num_attempts: 1)
21:04:42.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:42.907138 23569 meta_cache.cc:788] Timed out: LookupRpcById { tablet: '01aa0e192c434b468ec7ed90d48563b3', attempt: 2 } failed: LookupRpcById timed out after deadline expired: LookupRpcById { tablet: '01aa0e192c434b468ec7ed90d48563b3', attempt: 2 } passed its deadline: Timed out: ConnectToClusterRpc(addrs: 127.22.168.62:37239,127.22.168.61:32947,127.22.168.60:33247, num_attempts: 96) passed its deadline: Not found: no leader found: ConnectToClusterRpc(addrs: 127.22.168.62:37239,127.22.168.61:32947,127.22.168.60:33247, num_attempts: 1)
21:04:42.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:42.909590 23569 master_proxy_rpc.cc:203] Re-attempting LookupRpcById request to leader Master (127.22.168.60:33247)
21:04:42.944 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:42.944 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:42.945 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:42.945 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:42.945 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:42.945 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:42.945 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:42.945 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:42.947 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:42.948 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=162), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [43ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [63ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [99ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [161ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [161ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [162ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [163ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, deferred=Deferred@398144747(state=PENDING, result=null, callback=(continuation of Deferred@54400071 after retry RPC after error@927775279) -> (continuation of Deferred@1495544331 after retry RPC after error@393628072) -> (continuation of Deferred@1687812909 after retry RPC after error@911648851) -> (continuation of Deferred@1318744312 after retry RPC after error@1795443594) -> (continuation of Deferred@1089439060 after retry RPC after error@1131981098) -> (continuation of Deferred@1593318413 after retry RPC after error@392778819), errback=(continuation of Deferred@54400071 after retry RPC after error@927775279) -> (continuation of Deferred@1495544331 after retry RPC after error@393628072) -> (continuation of Deferred@1687812909 after retry RPC after error@911648851) -> (continuation of Deferred@1318744312 after retry RPC after error@1795443594) -> (continuation of Deferred@1089439060 after retry RPC after error@1131981098) -> (continuation of Deferred@1593318413 after retry RPC after error@392778819)))
21:04:42.948 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=162), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [43ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [63ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [99ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [161ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [161ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [162ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [163ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, deferred=Deferred@398144747(state=PENDING, result=null, callback=(continuation of Deferred@54400071 after retry RPC after error@927775279) -> (continuation of Deferred@1495544331 after retry RPC after error@393628072) -> (continuation of Deferred@1687812909 after retry RPC after error@911648851) -> (continuation of Deferred@1318744312 after retry RPC after error@1795443594) -> (continuation of Deferred@1089439060 after retry RPC after error@1131981098) -> (continuation of Deferred@1593318413 after retry RPC after error@392778819), errback=(continuation of Deferred@54400071 after retry RPC after error@927775279) -> (continuation of Deferred@1495544331 after retry RPC after error@393628072) -> (continuation of Deferred@1687812909 after retry RPC after error@911648851) -> (continuation of Deferred@1318744312 after retry RPC after error@1795443594) -> (continuation of Deferred@1089439060 after retry RPC after error@1131981098) -> (continuation of Deferred@1593318413 after retry RPC after error@392778819)))
21:04:42.964 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:42.964 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:42.965 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:42.965 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:42.965 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:42.965 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:42.965 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:42.965 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:42.966 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:42.984 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:42.984 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:42.985 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:42.985 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:42.985 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:42.985 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:42.985 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:42.985 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:42.986 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:43.004 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:43.004 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:43.005 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:43.005 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:43.005 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:43.005 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:43.005 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:43.005 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:43.007 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:43.024 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:43.024 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:43.025 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:43.025 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:43.025 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:43.025 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:43.025 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:43.025 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:43.026 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:43.044 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:43.044 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:43.045 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:43.045 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:43.045 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:43.045 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:43.045 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:43.045 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:43.046 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:43.084 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:43.084 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:43.085 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:43.085 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:43.085 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:43.085 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:43.085 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:43.085 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:43.086 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:43.144 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:43.145 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:43.145 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:43.145 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:43.145 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:43.146 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:43.146 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:43.146 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:43.146 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:43.205 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:43.205 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:43.205 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:43.206 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:43.206 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:43.206 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:43.206 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:43.206 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:43.207 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:43.207 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=242), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [181ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [182ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [182ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [242ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [242ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [242ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [243ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, deferred=Deferred@1995632062(state=PENDING, result=null, callback=(continuation of Deferred@608262626 after retry RPC after error@88106431) -> (continuation of Deferred@724921301 after retry RPC after error@1246489616) -> (continuation of Deferred@1693135602 after retry RPC after error@1652612685) -> (continuation of Deferred@1651206667 after retry RPC after error@2010456378) -> (continuation of Deferred@1242882448 after retry RPC after error@780282921) -> (continuation of Deferred@394277056 after retry RPC after error@1224871931) -> (continuation of Deferred@620195626 after retry RPC after error@464252153), errback=(continuation of Deferred@608262626 after retry RPC after error@88106431) -> (continuation of Deferred@724921301 after retry RPC after error@1246489616) -> (continuation of Deferred@1693135602 after retry RPC after error@1652612685) -> (continuation of Deferred@1651206667 after retry RPC after error@2010456378) -> (continuation of Deferred@1242882448 after retry RPC after error@780282921) -> (continuation of Deferred@394277056 after retry RPC after error@1224871931) -> (continuation of Deferred@620195626 after retry RPC after error@464252153)))
21:04:43.207 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=242), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [181ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [182ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [182ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [242ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [242ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [242ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [243ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, deferred=Deferred@1995632062(state=PENDING, result=null, callback=(continuation of Deferred@608262626 after retry RPC after error@88106431) -> (continuation of Deferred@724921301 after retry RPC after error@1246489616) -> (continuation of Deferred@1693135602 after retry RPC after error@1652612685) -> (continuation of Deferred@1651206667 after retry RPC after error@2010456378) -> (continuation of Deferred@1242882448 after retry RPC after error@780282921) -> (continuation of Deferred@394277056 after retry RPC after error@1224871931) -> (continuation of Deferred@620195626 after retry RPC after error@464252153), errback=(continuation of Deferred@608262626 after retry RPC after error@88106431) -> (continuation of Deferred@724921301 after retry RPC after error@1246489616) -> (continuation of Deferred@1693135602 after retry RPC after error@1652612685) -> (continuation of Deferred@1651206667 after retry RPC after error@2010456378) -> (continuation of Deferred@1242882448 after retry RPC after error@780282921) -> (continuation of Deferred@394277056 after retry RPC after error@1224871931) -> (continuation of Deferred@620195626 after retry RPC after error@464252153)))
21:04:43.225 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:43.225 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:43.225 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:43.226 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:43.226 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:43.226 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:43.227 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:43.227 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:43.227 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:43.244 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:43.245 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:43.245 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:43.245 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:43.245 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:43.245 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:43.246 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:43.246 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:43.246 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:43.264 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:43.265 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:43.265 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:43.265 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:43.266 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:43.265 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:43.266 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:43.266 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:43.266 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:43.284 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:43.285 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:43.285 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:43.285 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:43.285 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:43.285 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:43.286 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:43.286 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:43.286 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:43.304 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:43.304 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:43.305 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:43.305 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:43.305 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:43.305 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:43.305 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:43.305 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:43.306 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:43.344 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:43.344 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:43.345 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:43.345 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:43.345 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:43.346 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:43.346 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:43.346 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:43.346 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:43.364 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:43.364 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:43.365 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:43.365 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:43.365 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:43.365 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:43.365 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:43.365 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:43.366 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:43.424 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:43.424 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:43.425 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:43.425 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:43.425 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:43.425 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:43.425 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:43.425 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:43.426 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:43.464 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:43.466 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:43.466 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:43.467 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:43.467 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:43.467 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:43.467 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:43.467 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:43.467 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:43.468 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=9, TimeoutTracker(timeout=250, elapsed=243), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [3ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [3ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [141ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [141ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [142ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [201ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [201ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [202ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [243ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [243ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [243ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [243ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, deferred=Deferred@1787831994(state=PENDING, result=null, callback=(continuation of Deferred@2120893684 after retry RPC after error@1995466582) -> (continuation of Deferred@844789530 after retry RPC after error@1494491677) -> (continuation of Deferred@1979027503 after retry RPC after error@13271824) -> (continuation of Deferred@1640617652 after retry RPC after error@431297626) -> (continuation of Deferred@1075350443 after retry RPC after error@337527483) -> (continuation of Deferred@994921210 after retry RPC after error@183213581) -> (continuation of Deferred@246028584 after retry RPC after error@1749522908) -> (continuation of Deferred@957093044 after retry RPC after error@1874984558), errback=(continuation of Deferred@2120893684 after retry RPC after error@1995466582) -> (continuation of Deferred@844789530 after retry RPC after error@1494491677) -> (continuation of Deferred@1979027503 after retry RPC after error@13271824) -> (continuation of Deferred@1640617652 after retry RPC after error@431297626) -> (continuation of Deferred@1075350443 after retry RPC after error@337527483) -> (continuation of Deferred@994921210 after retry RPC after error@183213581) -> (continuation of Deferred@246028584 after retry RPC after error@1749522908) -> (continuation of Deferred@957093044 after retry RPC after error@1874984558)))
21:04:43.468 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=9, TimeoutTracker(timeout=250, elapsed=243), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [3ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [3ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [141ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [141ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [142ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [201ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [201ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [202ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [243ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [243ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [243ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [243ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, deferred=Deferred@1787831994(state=PENDING, result=null, callback=(continuation of Deferred@2120893684 after retry RPC after error@1995466582) -> (continuation of Deferred@844789530 after retry RPC after error@1494491677) -> (continuation of Deferred@1979027503 after retry RPC after error@13271824) -> (continuation of Deferred@1640617652 after retry RPC after error@431297626) -> (continuation of Deferred@1075350443 after retry RPC after error@337527483) -> (continuation of Deferred@994921210 after retry RPC after error@183213581) -> (continuation of Deferred@246028584 after retry RPC after error@1749522908) -> (continuation of Deferred@957093044 after retry RPC after error@1874984558), errback=(continuation of Deferred@2120893684 after retry RPC after error@1995466582) -> (continuation of Deferred@844789530 after retry RPC after error@1494491677) -> (continuation of Deferred@1979027503 after retry RPC after error@13271824) -> (continuation of Deferred@1640617652 after retry RPC after error@431297626) -> (continuation of Deferred@1075350443 after retry RPC after error@337527483) -> (continuation of Deferred@994921210 after retry RPC after error@183213581) -> (continuation of Deferred@246028584 after retry RPC after error@1749522908) -> (continuation of Deferred@957093044 after retry RPC after error@1874984558)))
21:04:43.484 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:43.484 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:43.485 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:43.485 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:43.485 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:43.485 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:43.485 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:43.485 [INFO - pool-40-thread-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:43.486 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:43.504 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:43.505 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:43.505 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:43.505 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:43.506 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:43.506 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:43.506 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:43.506 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:43.506 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:43.506 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:43.507 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:43.507 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:43.524 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:43.525 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:43.525 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:43.526 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:43.526 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:43.525 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:43.526 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:43.526 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:43.527 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:43.544 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:43.545 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:43.545 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:43.545 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:43.545 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:43.545 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:43.545 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:43.545 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:43.547 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:43.564 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:43.565 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:43.565 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:43.565 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:43.565 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:43.565 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:43.565 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:43.566 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:43.567 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:43.584 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:43.585 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:43.585 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:43.585 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:43.585 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:43.585 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:43.585 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:43.585 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:43.586 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:43.665 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:43.665 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:43.665 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:43.666 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:43.666 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:43.665 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:43.666 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:43.666 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:43.667 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:43.667 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=183), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [21ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [23ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [63ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [182ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [182ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [182ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [183ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, deferred=Deferred@906089661(state=PENDING, result=null, callback=(continuation of Deferred@2040307782 after retry RPC after error@985701875) -> (continuation of Deferred@1129610941 after retry RPC after error@2076769878) -> (continuation of Deferred@1741352953 after retry RPC after error@1499187702) -> (continuation of Deferred@966452726 after retry RPC after error@85755613) -> (continuation of Deferred@899224736 after retry RPC after error@847937436) -> (continuation of Deferred@1548851566 after retry RPC after error@1033557591), errback=(continuation of Deferred@2040307782 after retry RPC after error@985701875) -> (continuation of Deferred@1129610941 after retry RPC after error@2076769878) -> (continuation of Deferred@1741352953 after retry RPC after error@1499187702) -> (continuation of Deferred@966452726 after retry RPC after error@85755613) -> (continuation of Deferred@899224736 after retry RPC after error@847937436) -> (continuation of Deferred@1548851566 after retry RPC after error@1033557591)))
21:04:43.667 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=183), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [21ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [23ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [63ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [182ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [182ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [182ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [183ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, deferred=Deferred@906089661(state=PENDING, result=null, callback=(continuation of Deferred@2040307782 after retry RPC after error@985701875) -> (continuation of Deferred@1129610941 after retry RPC after error@2076769878) -> (continuation of Deferred@1741352953 after retry RPC after error@1499187702) -> (continuation of Deferred@966452726 after retry RPC after error@85755613) -> (continuation of Deferred@899224736 after retry RPC after error@847937436) -> (continuation of Deferred@1548851566 after retry RPC after error@1033557591), errback=(continuation of Deferred@2040307782 after retry RPC after error@985701875) -> (continuation of Deferred@1129610941 after retry RPC after error@2076769878) -> (continuation of Deferred@1741352953 after retry RPC after error@1499187702) -> (continuation of Deferred@966452726 after retry RPC after error@85755613) -> (continuation of Deferred@899224736 after retry RPC after error@847937436) -> (continuation of Deferred@1548851566 after retry RPC after error@1033557591)))
21:04:43.684 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:43.685 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:43.685 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:43.685 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:43.685 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:43.685 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:43.685 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:43.686 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:43.686 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:43.704 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:43.705 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:43.705 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:43.705 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:43.705 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:43.705 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:43.705 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:43.705 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:43.706 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:43.724 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:43.724 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:43.725 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:43.725 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:43.725 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:43.725 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:43.727 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:43.728 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:43.729 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:43.744 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:43.744 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:43.745 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:43.745 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:43.745 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:43.745 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:43.746 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:43.746 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:43.746 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:43.764 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:43.764 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:43.765 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:43.765 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:43.765 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:43.765 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:43.765 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:43.766 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:43.766 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:43.784 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:43.784 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:43.785 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:43.785 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:43.785 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:43.785 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:43.785 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:43.786 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:43.786 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:43.844 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:43.844 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:43.845 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:43.845 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:43.845 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:43.845 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:43.845 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:43.845 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:43.846 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:43.924 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:43.925 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:43.925 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:43.926 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:43.926 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:43.925 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:43.926 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:43.926 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:43.927 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:43.927 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=242), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [43ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [44ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [45ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [99ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [161ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [161ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [162ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [242ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [242ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [243ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [243ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, deferred=Deferred@133554073(state=PENDING, result=null, callback=(continuation of Deferred@1447817327 after retry RPC after error@1463253794) -> (continuation of Deferred@426886271 after retry RPC after error@819059167) -> (continuation of Deferred@965775138 after retry RPC after error@1435635033) -> (continuation of Deferred@1113029032 after retry RPC after error@2088128284) -> (continuation of Deferred@1092761135 after retry RPC after error@77534103) -> (continuation of Deferred@2113708181 after retry RPC after error@859573913) -> (continuation of Deferred@1669318047 after retry RPC after error@520118732), errback=(continuation of Deferred@1447817327 after retry RPC after error@1463253794) -> (continuation of Deferred@426886271 after retry RPC after error@819059167) -> (continuation of Deferred@965775138 after retry RPC after error@1435635033) -> (continuation of Deferred@1113029032 after retry RPC after error@2088128284) -> (continuation of Deferred@1092761135 after retry RPC after error@77534103) -> (continuation of Deferred@2113708181 after retry RPC after error@859573913) -> (continuation of Deferred@1669318047 after retry RPC after error@520118732)))
21:04:43.928 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=242), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [43ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [44ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [45ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [99ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [161ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [161ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [162ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [242ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [242ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [243ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [243ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, deferred=Deferred@133554073(state=PENDING, result=null, callback=(continuation of Deferred@1447817327 after retry RPC after error@1463253794) -> (continuation of Deferred@426886271 after retry RPC after error@819059167) -> (continuation of Deferred@965775138 after retry RPC after error@1435635033) -> (continuation of Deferred@1113029032 after retry RPC after error@2088128284) -> (continuation of Deferred@1092761135 after retry RPC after error@77534103) -> (continuation of Deferred@2113708181 after retry RPC after error@859573913) -> (continuation of Deferred@1669318047 after retry RPC after error@520118732), errback=(continuation of Deferred@1447817327 after retry RPC after error@1463253794) -> (continuation of Deferred@426886271 after retry RPC after error@819059167) -> (continuation of Deferred@965775138 after retry RPC after error@1435635033) -> (continuation of Deferred@1113029032 after retry RPC after error@2088128284) -> (continuation of Deferred@1092761135 after retry RPC after error@77534103) -> (continuation of Deferred@2113708181 after retry RPC after error@859573913) -> (continuation of Deferred@1669318047 after retry RPC after error@520118732)))
21:04:43.944 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:43.945 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:43.945 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:43.945 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:43.945 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:43.946 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:43.946 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:43.946 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:43.946 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:43.964 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:43.965 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:43.965 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:43.965 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:43.965 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:43.965 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:43.965 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:43.965 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:43.966 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:43.984 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:43.984 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:43.985 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:43.985 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:43.985 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:43.985 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:43.985 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:43.985 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:43.986 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:44.004 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:44.004 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:44.005 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:44.005 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:44.005 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:44.005 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:44.005 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:44.005 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:44.006 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:44.024 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:44.024 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:44.025 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:44.025 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:44.025 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:44.025 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:44.025 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:44.025 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:44.026 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:44.044 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:44.044 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:44.045 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:44.045 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:44.045 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:44.045 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:44.045 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:44.045 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:44.046 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:44.064 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:44.065 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:44.065 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:44.065 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:44.065 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:44.065 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:44.066 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:44.066 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:44.067 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:44.125 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:44.125 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:44.126 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:44.126 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:44.126 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:44.126 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:44.126 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:44.126 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:44.127 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:44.127 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=182), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [99ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [182ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [182ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [183ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [183ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, deferred=Deferred@479834776(state=PENDING, result=null, callback=(continuation of Deferred@1461346031 after retry RPC after error@982360747) -> (continuation of Deferred@1601682669 after retry RPC after error@246139113) -> (continuation of Deferred@188945141 after retry RPC after error@1552703948) -> (continuation of Deferred@1056849508 after retry RPC after error@1858229714) -> (continuation of Deferred@2105878085 after retry RPC after error@1119398456) -> (continuation of Deferred@226140437 after retry RPC after error@1911163309) -> (continuation of Deferred@621391579 after retry RPC after error@585578792), errback=(continuation of Deferred@1461346031 after retry RPC after error@982360747) -> (continuation of Deferred@1601682669 after retry RPC after error@246139113) -> (continuation of Deferred@188945141 after retry RPC after error@1552703948) -> (continuation of Deferred@1056849508 after retry RPC after error@1858229714) -> (continuation of Deferred@2105878085 after retry RPC after error@1119398456) -> (continuation of Deferred@226140437 after retry RPC after error@1911163309) -> (continuation of Deferred@621391579 after retry RPC after error@585578792)))
21:04:44.128 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=182), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [99ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [182ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [182ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [183ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [183ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, deferred=Deferred@479834776(state=PENDING, result=null, callback=(continuation of Deferred@1461346031 after retry RPC after error@982360747) -> (continuation of Deferred@1601682669 after retry RPC after error@246139113) -> (continuation of Deferred@188945141 after retry RPC after error@1552703948) -> (continuation of Deferred@1056849508 after retry RPC after error@1858229714) -> (continuation of Deferred@2105878085 after retry RPC after error@1119398456) -> (continuation of Deferred@226140437 after retry RPC after error@1911163309) -> (continuation of Deferred@621391579 after retry RPC after error@585578792), errback=(continuation of Deferred@1461346031 after retry RPC after error@982360747) -> (continuation of Deferred@1601682669 after retry RPC after error@246139113) -> (continuation of Deferred@188945141 after retry RPC after error@1552703948) -> (continuation of Deferred@1056849508 after retry RPC after error@1858229714) -> (continuation of Deferred@2105878085 after retry RPC after error@1119398456) -> (continuation of Deferred@226140437 after retry RPC after error@1911163309) -> (continuation of Deferred@621391579 after retry RPC after error@585578792)))
21:04:44.144 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:44.144 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:44.144 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:44.144 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:44.145 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:44.145 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:44.145 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:44.145 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:44.146 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:44.164 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:44.165 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:44.165 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:44.165 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:44.165 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:44.165 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:44.166 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:44.166 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:44.167 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:44.185 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:44.185 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:44.186 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:44.186 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:44.186 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:44.186 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:44.186 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:44.186 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:44.187 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:44.204 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:44.204 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:44.204 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:44.204 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:44.205 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:44.205 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:44.205 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:44.205 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:44.206 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:44.224 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:44.225 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:44.225 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:44.225 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:44.226 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:44.226 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:44.226 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:44.226 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:44.227 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:44.264 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:44.265 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:44.265 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:44.265 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:44.266 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:44.266 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:44.266 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:44.266 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:44.266 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:44.304 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:44.304 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:44.305 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:44.305 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:44.305 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:44.305 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:44.305 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:44.305 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:44.306 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:44.404 [DEBUG - pool-40-thread-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=259), Traces: [0ms] refreshing cache from master, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [3ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [21ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [22ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [22ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [23ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [23ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [24ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [41ms] refreshing cache from master, [42ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [42ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [42ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [43ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [43ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [43ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [44ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [81ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [82ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [82ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [83ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [83ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [83ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [84ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [121ms] refreshing cache from master, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [122ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [122ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [123ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [123ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [123ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [161ms] refreshing cache from master, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [162ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [162ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [163ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [163ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, deferred=Deferred@63890021(state=PENDING, result=null, callback=(continuation of Deferred@635231564 after retry RPC after error@587858425) -> (continuation of Deferred@1117752486 after retry RPC after error@909546330) -> (continuation of Deferred@1975721581 after retry RPC after error@112577765) -> (continuation of Deferred@955357737 after retry RPC after error@987722286) -> (continuation of Deferred@1459757934 after retry RPC after error@2146967439) -> (continuation of Deferred@1265427083 after retry RPC after error@174307274) -> (continuation of Deferred@1279061179 after retry RPC after error@1938265812), errback=(continuation of Deferred@635231564 after retry RPC after error@587858425) -> (continuation of Deferred@1117752486 after retry RPC after error@909546330) -> (continuation of Deferred@1975721581 after retry RPC after error@112577765) -> (continuation of Deferred@955357737 after retry RPC after error@987722286) -> (continuation of Deferred@1459757934 after retry RPC after error@2146967439) -> (continuation of Deferred@1265427083 after retry RPC after error@174307274) -> (continuation of Deferred@1279061179 after retry RPC after error@1938265812)))
21:04:44.404 [WARN - pool-40-thread-1] (HashedWheelTimer.java:716) An exception was thrown by TimerTask.
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343) ~[main/:?]
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at java.lang.Thread.run(Thread.java:748) [?:1.8.0_275]
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86)
at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106)
at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198)
at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158)
at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953)
at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923)
at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464)
at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713)
at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701)
at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788)
at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501)
at java.lang.Thread.run(Thread.java:748)
21:04:44.424 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:44.425 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:44.425 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:44.425 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:44.425 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:44.425 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:44.425 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:44.425 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:44.426 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:44.444 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:44.444 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:44.445 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:44.445 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:44.445 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:44.445 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:44.445 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:44.445 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:44.446 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:44.464 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:44.464 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:44.465 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:44.465 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:44.465 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:44.465 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:44.465 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:44.466 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:44.467 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:44.484 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:44.484 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:44.485 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:44.485 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:44.485 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:44.485 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:44.485 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:44.485 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:44.486 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:44.504 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:44.504 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:44.505 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:44.505 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:44.505 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:44.505 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:44.505 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:44.505 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:44.506 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:44.544 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:44.544 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:44.545 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:44.545 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:44.545 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:44.545 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:44.545 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:44.545 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:44.546 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:44.604 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:44.604 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:44.605 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:44.605 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:44.605 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:44.605 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:44.605 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:44.605 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:44.606 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:44.607 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=182), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [43ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [181ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [181ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [182ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, deferred=Deferred@790661775(state=PENDING, result=null, callback=(continuation of Deferred@1440848299 after retry RPC after error@193925821) -> (continuation of Deferred@1485715659 after retry RPC after error@1934734759) -> (continuation of Deferred@707020455 after retry RPC after error@1506653092) -> (continuation of Deferred@1890357302 after retry RPC after error@237956873) -> (continuation of Deferred@112689429 after retry RPC after error@852710132) -> (continuation of Deferred@652888504 after retry RPC after error@1606790347), errback=(continuation of Deferred@1440848299 after retry RPC after error@193925821) -> (continuation of Deferred@1485715659 after retry RPC after error@1934734759) -> (continuation of Deferred@707020455 after retry RPC after error@1506653092) -> (continuation of Deferred@1890357302 after retry RPC after error@237956873) -> (continuation of Deferred@112689429 after retry RPC after error@852710132) -> (continuation of Deferred@652888504 after retry RPC after error@1606790347)))
21:04:44.607 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=182), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [43ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [181ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [181ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [182ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, deferred=Deferred@790661775(state=PENDING, result=null, callback=(continuation of Deferred@1440848299 after retry RPC after error@193925821) -> (continuation of Deferred@1485715659 after retry RPC after error@1934734759) -> (continuation of Deferred@707020455 after retry RPC after error@1506653092) -> (continuation of Deferred@1890357302 after retry RPC after error@237956873) -> (continuation of Deferred@112689429 after retry RPC after error@852710132) -> (continuation of Deferred@652888504 after retry RPC after error@1606790347), errback=(continuation of Deferred@1440848299 after retry RPC after error@193925821) -> (continuation of Deferred@1485715659 after retry RPC after error@1934734759) -> (continuation of Deferred@707020455 after retry RPC after error@1506653092) -> (continuation of Deferred@1890357302 after retry RPC after error@237956873) -> (continuation of Deferred@112689429 after retry RPC after error@852710132) -> (continuation of Deferred@652888504 after retry RPC after error@1606790347)))
21:04:44.624 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:44.624 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:44.625 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:44.625 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:44.625 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:44.625 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:44.625 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:44.625 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:44.626 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:44.644 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:44.644 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:44.645 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:44.645 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:44.645 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:44.645 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:44.645 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:44.645 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:44.646 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:44.664 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:44.664 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:44.665 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:44.665 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:44.665 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:44.665 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:44.665 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:44.665 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:44.666 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:44.684 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:44.684 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:44.684 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:44.684 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:44.685 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:44.685 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:44.685 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:44.685 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:44.686 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:44.704 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:44.705 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:44.705 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:44.705 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:44.705 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:44.706 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:44.706 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:44.706 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:44.706 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:44.745 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:44.745 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:44.745 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:44.745 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:44.746 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:44.746 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:44.746 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:44.746 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:44.746 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:44.784 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:44.785 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:44.785 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:44.785 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:44.785 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:44.785 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:44.786 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:44.786 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:44.786 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:44.884 [DEBUG - pool-40-thread-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=259), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [39ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [59ms] refreshing cache from master, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [162ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [162ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [162ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, deferred=Deferred@1661287064(state=PENDING, result=null, callback=(continuation of Deferred@516142213 after retry RPC after error@1748319193) -> (continuation of Deferred@222183061 after retry RPC after error@1201996880) -> (continuation of Deferred@1153558461 after retry RPC after error@90957583) -> (continuation of Deferred@2044570529 after retry RPC after error@381007694) -> (continuation of Deferred@1568818730 after retry RPC after error@118198538) -> (continuation of Deferred@329233129 after retry RPC after error@673937427) -> (continuation of Deferred@1733806122 after retry RPC after error@1016038923), errback=(continuation of Deferred@516142213 after retry RPC after error@1748319193) -> (continuation of Deferred@222183061 after retry RPC after error@1201996880) -> (continuation of Deferred@1153558461 after retry RPC after error@90957583) -> (continuation of Deferred@2044570529 after retry RPC after error@381007694) -> (continuation of Deferred@1568818730 after retry RPC after error@118198538) -> (continuation of Deferred@329233129 after retry RPC after error@673937427) -> (continuation of Deferred@1733806122 after retry RPC after error@1016038923)))
21:04:44.885 [WARN - pool-40-thread-1] (HashedWheelTimer.java:716) An exception was thrown by TimerTask.
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343) ~[main/:?]
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at java.lang.Thread.run(Thread.java:748) [?:1.8.0_275]
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86)
at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106)
at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198)
at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158)
at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953)
at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923)
at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464)
at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713)
at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701)
at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788)
at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501)
at java.lang.Thread.run(Thread.java:748)
21:04:44.904 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:44.904 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:44.905 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:44.905 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:44.905 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:44.905 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:44.905 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:44.905 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:44.906 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:44.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:44.921002 24071 master_runner.cc:186] Error getting master registration for 127.22.168.62:37239: Network error: Client connection negotiation failed: client connection to 127.22.168.62:37239: connect: Connection refused (error 111),
21:04:44.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:44.921248 24071 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
21:04:44.924 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:44.924 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:44.924 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:44.925 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:44.925 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:44.925 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:44.925 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:44.925 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:44.925 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:44.944 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:44.944 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:44.945 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:44.945 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:44.945 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:44.945 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:44.945 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:44.945 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:44.946 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:44.964 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:44.964 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:44.965 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:44.965 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:44.965 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:44.965 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:44.965 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:44.965 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:44.966 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:44.984 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:44.985 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:44.986 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:44.986 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:44.986 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:44.986 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:44.986 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:44.987 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:44.987 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:45.004 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:45.004 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:45.005 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:45.005 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:45.005 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:45.005 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:45.005 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:45.005 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:45.006 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:45.064 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:45.065 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:45.065 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:45.065 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:45.065 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:45.065 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:45.066 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:45.066 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:45.066 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:45.067 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=162), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [19ms] refreshing cache from master, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [79ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [83ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [161ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [161ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [162ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, deferred=Deferred@451535009(state=PENDING, result=null, callback=(continuation of Deferred@584900934 after retry RPC after error@292814188) -> (continuation of Deferred@1648360243 after retry RPC after error@931590973) -> (continuation of Deferred@1914040922 after retry RPC after error@2080635369) -> (continuation of Deferred@1635279673 after retry RPC after error@1634167600) -> (continuation of Deferred@1010680420 after retry RPC after error@1595058363) -> (continuation of Deferred@2110876107 after retry RPC after error@666215338), errback=(continuation of Deferred@584900934 after retry RPC after error@292814188) -> (continuation of Deferred@1648360243 after retry RPC after error@931590973) -> (continuation of Deferred@1914040922 after retry RPC after error@2080635369) -> (continuation of Deferred@1635279673 after retry RPC after error@1634167600) -> (continuation of Deferred@1010680420 after retry RPC after error@1595058363) -> (continuation of Deferred@2110876107 after retry RPC after error@666215338)))
21:04:45.068 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=162), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [19ms] refreshing cache from master, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [79ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [83ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [161ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [161ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [162ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, deferred=Deferred@451535009(state=PENDING, result=null, callback=(continuation of Deferred@584900934 after retry RPC after error@292814188) -> (continuation of Deferred@1648360243 after retry RPC after error@931590973) -> (continuation of Deferred@1914040922 after retry RPC after error@2080635369) -> (continuation of Deferred@1635279673 after retry RPC after error@1634167600) -> (continuation of Deferred@1010680420 after retry RPC after error@1595058363) -> (continuation of Deferred@2110876107 after retry RPC after error@666215338), errback=(continuation of Deferred@584900934 after retry RPC after error@292814188) -> (continuation of Deferred@1648360243 after retry RPC after error@931590973) -> (continuation of Deferred@1914040922 after retry RPC after error@2080635369) -> (continuation of Deferred@1635279673 after retry RPC after error@1634167600) -> (continuation of Deferred@1010680420 after retry RPC after error@1595058363) -> (continuation of Deferred@2110876107 after retry RPC after error@666215338)))
21:04:45.084 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:45.085 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:45.085 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:45.085 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:45.085 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:45.085 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:45.085 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:45.085 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:45.086 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:45.104 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:45.104 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:45.105 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:45.105 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:45.105 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:45.105 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:45.105 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:45.105 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:45.106 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:45.124 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:45.124 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:45.124 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:45.124 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:45.125 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:45.125 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:45.125 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:45.125 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:45.126 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:45.144 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:45.145 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:45.145 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:45.145 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:45.145 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:45.145 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:45.145 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:45.145 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:45.146 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:45.164 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:45.164 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:45.165 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:45.165 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:45.165 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:45.165 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:45.165 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:45.165 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:45.166 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:45.184 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:45.184 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:45.185 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:45.185 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:45.185 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:45.185 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:45.185 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:45.185 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:45.186 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:45.224 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:45.224 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:45.225 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:45.225 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:45.225 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:45.225 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:45.225 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:45.225 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:45.226 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:45.264 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:45.264 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:45.265 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:45.265 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:45.265 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:45.265 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:45.265 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:45.265 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:45.266 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:45.304 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:45.304 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:45.304 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:45.305 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:45.305 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:45.305 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:45.305 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:45.305 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:45.306 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:45.306 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=9, TimeoutTracker(timeout=250, elapsed=221), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [39ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [141ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [141ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [142ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [179ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [181ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [181ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [181ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [221ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [221ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [221ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [222ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, deferred=Deferred@2107506236(state=PENDING, result=null, callback=(continuation of Deferred@1927651976 after retry RPC after error@1398086061) -> (continuation of Deferred@1093343601 after retry RPC after error@1406493558) -> (continuation of Deferred@209414758 after retry RPC after error@2006487605) -> (continuation of Deferred@1851915558 after retry RPC after error@383253839) -> (continuation of Deferred@1101569449 after retry RPC after error@1922217619) -> (continuation of Deferred@335277354 after retry RPC after error@988764063) -> (continuation of Deferred@1060094142 after retry RPC after error@1483661286) -> (continuation of Deferred@338750332 after retry RPC after error@1411585482), errback=(continuation of Deferred@1927651976 after retry RPC after error@1398086061) -> (continuation of Deferred@1093343601 after retry RPC after error@1406493558) -> (continuation of Deferred@209414758 after retry RPC after error@2006487605) -> (continuation of Deferred@1851915558 after retry RPC after error@383253839) -> (continuation of Deferred@1101569449 after retry RPC after error@1922217619) -> (continuation of Deferred@335277354 after retry RPC after error@988764063) -> (continuation of Deferred@1060094142 after retry RPC after error@1483661286) -> (continuation of Deferred@338750332 after retry RPC after error@1411585482)))
21:04:45.307 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=9, TimeoutTracker(timeout=250, elapsed=221), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [39ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [141ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [141ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [142ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [179ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [181ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [181ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [181ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [221ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [221ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [221ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [222ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, deferred=Deferred@2107506236(state=PENDING, result=null, callback=(continuation of Deferred@1927651976 after retry RPC after error@1398086061) -> (continuation of Deferred@1093343601 after retry RPC after error@1406493558) -> (continuation of Deferred@209414758 after retry RPC after error@2006487605) -> (continuation of Deferred@1851915558 after retry RPC after error@383253839) -> (continuation of Deferred@1101569449 after retry RPC after error@1922217619) -> (continuation of Deferred@335277354 after retry RPC after error@988764063) -> (continuation of Deferred@1060094142 after retry RPC after error@1483661286) -> (continuation of Deferred@338750332 after retry RPC after error@1411585482), errback=(continuation of Deferred@1927651976 after retry RPC after error@1398086061) -> (continuation of Deferred@1093343601 after retry RPC after error@1406493558) -> (continuation of Deferred@209414758 after retry RPC after error@2006487605) -> (continuation of Deferred@1851915558 after retry RPC after error@383253839) -> (continuation of Deferred@1101569449 after retry RPC after error@1922217619) -> (continuation of Deferred@335277354 after retry RPC after error@988764063) -> (continuation of Deferred@1060094142 after retry RPC after error@1483661286) -> (continuation of Deferred@338750332 after retry RPC after error@1411585482)))
21:04:45.325 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:45.325 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:45.325 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:45.325 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:45.326 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:45.326 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:45.326 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:45.326 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:45.327 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:45.344 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:45.345 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:45.345 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:45.345 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:45.345 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:45.345 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:45.346 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:45.346 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:45.347 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:45.364 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:45.365 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:45.365 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:45.365 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:45.365 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:45.365 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:45.365 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:45.365 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:45.366 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:45.384 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:45.384 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:45.385 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:45.385 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:45.385 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:45.385 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:45.385 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:45.385 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:45.386 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:45.404 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:45.404 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:45.405 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:45.405 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:45.405 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:45.405 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:45.405 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:45.405 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:45.406 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:45.444 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:45.444 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:45.445 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:45.445 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:45.445 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:45.445 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:45.445 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:45.445 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:45.446 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:45.504 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:45.505 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:45.505 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:45.505 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:45.506 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:45.506 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:45.506 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:45.506 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:45.506 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:45.507 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=182), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [182ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [182ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [182ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, deferred=Deferred@1033542448(state=PENDING, result=null, callback=(continuation of Deferred@541028672 after retry RPC after error@687136246) -> (continuation of Deferred@1213670998 after retry RPC after error@561870711) -> (continuation of Deferred@2048112489 after retry RPC after error@1946607280) -> (continuation of Deferred@93467632 after retry RPC after error@1335769850) -> (continuation of Deferred@676426911 after retry RPC after error@1784728699) -> (continuation of Deferred@2130280308 after retry RPC after error@1083098043), errback=(continuation of Deferred@541028672 after retry RPC after error@687136246) -> (continuation of Deferred@1213670998 after retry RPC after error@561870711) -> (continuation of Deferred@2048112489 after retry RPC after error@1946607280) -> (continuation of Deferred@93467632 after retry RPC after error@1335769850) -> (continuation of Deferred@676426911 after retry RPC after error@1784728699) -> (continuation of Deferred@2130280308 after retry RPC after error@1083098043)))
21:04:45.507 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=182), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [182ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [182ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [182ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, deferred=Deferred@1033542448(state=PENDING, result=null, callback=(continuation of Deferred@541028672 after retry RPC after error@687136246) -> (continuation of Deferred@1213670998 after retry RPC after error@561870711) -> (continuation of Deferred@2048112489 after retry RPC after error@1946607280) -> (continuation of Deferred@93467632 after retry RPC after error@1335769850) -> (continuation of Deferred@676426911 after retry RPC after error@1784728699) -> (continuation of Deferred@2130280308 after retry RPC after error@1083098043), errback=(continuation of Deferred@541028672 after retry RPC after error@687136246) -> (continuation of Deferred@1213670998 after retry RPC after error@561870711) -> (continuation of Deferred@2048112489 after retry RPC after error@1946607280) -> (continuation of Deferred@93467632 after retry RPC after error@1335769850) -> (continuation of Deferred@676426911 after retry RPC after error@1784728699) -> (continuation of Deferred@2130280308 after retry RPC after error@1083098043)))
21:04:45.524 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:45.525 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:45.525 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:45.525 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:45.525 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:45.525 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:45.526 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:45.526 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:45.527 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:45.544 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:45.545 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:45.545 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:45.545 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:45.545 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:45.545 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:45.545 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:45.546 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:45.546 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:45.564 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:45.564 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:45.565 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:45.565 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:45.565 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:45.565 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:45.565 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:45.565 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:45.566 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:45.584 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:45.584 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:45.585 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:45.585 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:45.585 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:45.585 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:45.585 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:45.585 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:45.586 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:45.604 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:45.604 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:45.605 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:45.605 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:45.605 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:45.605 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:45.605 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:45.605 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:45.606 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:45.644 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:45.644 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:45.645 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:45.645 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:45.645 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:45.645 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:45.645 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:45.645 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:45.646 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:45.684 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:45.684 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:45.685 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:45.685 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:45.685 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:45.685 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:45.685 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:45.685 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:45.686 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:45.704 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:45.704 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:45.704 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:45.705 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:45.705 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:45.705 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:45.705 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:45.705 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:45.706 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:45.724 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:45.725 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:45.725 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:45.725 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:45.725 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:45.725 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:45.726 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:45.726 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:45.726 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:45.727 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=9, TimeoutTracker(timeout=250, elapsed=202), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [161ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [161ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [162ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [179ms] refreshing cache from master, [179ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [181ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [181ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [182ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [202ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [202ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [202ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, deferred=Deferred@1794416477(state=PENDING, result=null, callback=(continuation of Deferred@301444631 after retry RPC after error@1974697489) -> (continuation of Deferred@679359692 after retry RPC after error@1715362025) -> (continuation of Deferred@1251333423 after retry RPC after error@616069530) -> (continuation of Deferred@2100945301 after retry RPC after error@1775507403) -> (continuation of Deferred@227318643 after retry RPC after error@1788827633) -> (continuation of Deferred@1486370430 after retry RPC after error@297470185) -> (continuation of Deferred@2013613742 after retry RPC after error@1748352640) -> (continuation of Deferred@1896308286 after retry RPC after error@951250103), errback=(continuation of Deferred@301444631 after retry RPC after error@1974697489) -> (continuation of Deferred@679359692 after retry RPC after error@1715362025) -> (continuation of Deferred@1251333423 after retry RPC after error@616069530) -> (continuation of Deferred@2100945301 after retry RPC after error@1775507403) -> (continuation of Deferred@227318643 after retry RPC after error@1788827633) -> (continuation of Deferred@1486370430 after retry RPC after error@297470185) -> (continuation of Deferred@2013613742 after retry RPC after error@1748352640) -> (continuation of Deferred@1896308286 after retry RPC after error@951250103)))
21:04:45.727 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=9, TimeoutTracker(timeout=250, elapsed=202), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [161ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [161ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [162ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [179ms] refreshing cache from master, [179ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [181ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [181ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [182ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [202ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [202ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [202ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, deferred=Deferred@1794416477(state=PENDING, result=null, callback=(continuation of Deferred@301444631 after retry RPC after error@1974697489) -> (continuation of Deferred@679359692 after retry RPC after error@1715362025) -> (continuation of Deferred@1251333423 after retry RPC after error@616069530) -> (continuation of Deferred@2100945301 after retry RPC after error@1775507403) -> (continuation of Deferred@227318643 after retry RPC after error@1788827633) -> (continuation of Deferred@1486370430 after retry RPC after error@297470185) -> (continuation of Deferred@2013613742 after retry RPC after error@1748352640) -> (continuation of Deferred@1896308286 after retry RPC after error@951250103), errback=(continuation of Deferred@301444631 after retry RPC after error@1974697489) -> (continuation of Deferred@679359692 after retry RPC after error@1715362025) -> (continuation of Deferred@1251333423 after retry RPC after error@616069530) -> (continuation of Deferred@2100945301 after retry RPC after error@1775507403) -> (continuation of Deferred@227318643 after retry RPC after error@1788827633) -> (continuation of Deferred@1486370430 after retry RPC after error@297470185) -> (continuation of Deferred@2013613742 after retry RPC after error@1748352640) -> (continuation of Deferred@1896308286 after retry RPC after error@951250103)))
21:04:45.744 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:45.745 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:45.745 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:45.745 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:45.746 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:45.746 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:45.746 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:45.746 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:45.746 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:45.764 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:45.765 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:45.765 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:45.765 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:45.765 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:45.765 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:45.765 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:45.765 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:45.766 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:45.784 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:45.784 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:45.785 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:45.785 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:45.786 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:45.786 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:45.786 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:45.786 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:45.786 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:45.804 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:45.804 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:45.805 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:45.805 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:45.805 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:45.805 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:45.805 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:45.805 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:45.806 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:45.824 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:45.824 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:45.825 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:45.825 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:45.825 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:45.825 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:45.825 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:45.825 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:45.826 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:45.845 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:45.845 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:45.845 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:45.845 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:45.846 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:45.846 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:45.846 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:45.846 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:45.846 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:45.904 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:45.904 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:45.904 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:45.904 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:45.905 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:45.905 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:45.905 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:45.905 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:45.906 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:45.984 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:45.984 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:45.985 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:45.985 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:45.985 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:45.985 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:45.985 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:45.985 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:45.986 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:45.992 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=241), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [159ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [161ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [161ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [162ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [241ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [241ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [242ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [242ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, deferred=Deferred@919539628(state=PENDING, result=null, callback=(continuation of Deferred@622092552 after retry RPC after error@1263967145) -> (continuation of Deferred@787241948 after retry RPC after error@1611699138) -> (continuation of Deferred@1720365801 after retry RPC after error@436135490) -> (continuation of Deferred@1426095130 after retry RPC after error@837450747) -> (continuation of Deferred@46439125 after retry RPC after error@1818665017) -> (continuation of Deferred@986767619 after retry RPC after error@1485271041) -> (continuation of Deferred@2125683702 after retry RPC after error@663575409), errback=(continuation of Deferred@622092552 after retry RPC after error@1263967145) -> (continuation of Deferred@787241948 after retry RPC after error@1611699138) -> (continuation of Deferred@1720365801 after retry RPC after error@436135490) -> (continuation of Deferred@1426095130 after retry RPC after error@837450747) -> (continuation of Deferred@46439125 after retry RPC after error@1818665017) -> (continuation of Deferred@986767619 after retry RPC after error@1485271041) -> (continuation of Deferred@2125683702 after retry RPC after error@663575409)))
21:04:45.993 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=241), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [159ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [161ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [161ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [162ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [241ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [241ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [242ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [242ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, deferred=Deferred@919539628(state=PENDING, result=null, callback=(continuation of Deferred@622092552 after retry RPC after error@1263967145) -> (continuation of Deferred@787241948 after retry RPC after error@1611699138) -> (continuation of Deferred@1720365801 after retry RPC after error@436135490) -> (continuation of Deferred@1426095130 after retry RPC after error@837450747) -> (continuation of Deferred@46439125 after retry RPC after error@1818665017) -> (continuation of Deferred@986767619 after retry RPC after error@1485271041) -> (continuation of Deferred@2125683702 after retry RPC after error@663575409), errback=(continuation of Deferred@622092552 after retry RPC after error@1263967145) -> (continuation of Deferred@787241948 after retry RPC after error@1611699138) -> (continuation of Deferred@1720365801 after retry RPC after error@436135490) -> (continuation of Deferred@1426095130 after retry RPC after error@837450747) -> (continuation of Deferred@46439125 after retry RPC after error@1818665017) -> (continuation of Deferred@986767619 after retry RPC after error@1485271041) -> (continuation of Deferred@2125683702 after retry RPC after error@663575409)))
21:04:46.005 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:46.005 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:46.005 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:46.005 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:46.006 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:46.006 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:46.006 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:46.006 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:46.006 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:46.025 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:46.025 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:46.025 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:46.025 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:46.026 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:46.026 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:46.026 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:46.026 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:46.026 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:46.044 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:46.045 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:46.045 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:46.045 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:46.045 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:46.045 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:46.045 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:46.045 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:46.046 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:46.064 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:46.064 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:46.065 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:46.065 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:46.065 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:46.065 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:46.065 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:46.065 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:46.066 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:46.084 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:46.084 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:46.085 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:46.085 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:46.085 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:46.085 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:46.085 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:46.085 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:46.086 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:46.105 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:46.105 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:46.105 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:46.105 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:46.105 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:46.105 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:46.106 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:46.106 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:46.107 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:46.124 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:46.124 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:46.125 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:46.125 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:46.125 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:46.125 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:46.125 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:46.125 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:46.126 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:46.184 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:46.185 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:46.185 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:46.185 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:46.185 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:46.185 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:46.185 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:46.185 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:46.186 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:46.187 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=181), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [181ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [181ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [182ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, deferred=Deferred@1216593322(state=PENDING, result=null, callback=(continuation of Deferred@1105566021 after retry RPC after error@627267144) -> (continuation of Deferred@1684033054 after retry RPC after error@836489685) -> (continuation of Deferred@1545160986 after retry RPC after error@1668781511) -> (continuation of Deferred@92354313 after retry RPC after error@1782254357) -> (continuation of Deferred@2088554193 after retry RPC after error@595244340) -> (continuation of Deferred@802154285 after retry RPC after error@488514146) -> (continuation of Deferred@126778710 after retry RPC after error@1898691462), errback=(continuation of Deferred@1105566021 after retry RPC after error@627267144) -> (continuation of Deferred@1684033054 after retry RPC after error@836489685) -> (continuation of Deferred@1545160986 after retry RPC after error@1668781511) -> (continuation of Deferred@92354313 after retry RPC after error@1782254357) -> (continuation of Deferred@2088554193 after retry RPC after error@595244340) -> (continuation of Deferred@802154285 after retry RPC after error@488514146) -> (continuation of Deferred@126778710 after retry RPC after error@1898691462)))
21:04:46.187 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=181), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [181ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [181ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [182ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, deferred=Deferred@1216593322(state=PENDING, result=null, callback=(continuation of Deferred@1105566021 after retry RPC after error@627267144) -> (continuation of Deferred@1684033054 after retry RPC after error@836489685) -> (continuation of Deferred@1545160986 after retry RPC after error@1668781511) -> (continuation of Deferred@92354313 after retry RPC after error@1782254357) -> (continuation of Deferred@2088554193 after retry RPC after error@595244340) -> (continuation of Deferred@802154285 after retry RPC after error@488514146) -> (continuation of Deferred@126778710 after retry RPC after error@1898691462), errback=(continuation of Deferred@1105566021 after retry RPC after error@627267144) -> (continuation of Deferred@1684033054 after retry RPC after error@836489685) -> (continuation of Deferred@1545160986 after retry RPC after error@1668781511) -> (continuation of Deferred@92354313 after retry RPC after error@1782254357) -> (continuation of Deferred@2088554193 after retry RPC after error@595244340) -> (continuation of Deferred@802154285 after retry RPC after error@488514146) -> (continuation of Deferred@126778710 after retry RPC after error@1898691462)))
21:04:46.204 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:46.204 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:46.205 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:46.205 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:46.205 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:46.205 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:46.205 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:46.205 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:46.206 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:46.224 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:46.224 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:46.225 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:46.225 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:46.225 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:46.225 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:46.225 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:46.225 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:46.226 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:46.244 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:46.244 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:46.245 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:46.245 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:46.245 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:46.245 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:46.245 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:46.245 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:46.246 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:46.265 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:46.265 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:46.265 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:46.265 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:46.265 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:46.265 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:46.266 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:46.266 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:46.267 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:46.284 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:46.285 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:46.285 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:46.285 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:46.286 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:46.285 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:46.286 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:46.286 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:46.287 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:46.304 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:46.304 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:46.305 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:46.305 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:46.305 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:46.305 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:46.305 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:46.305 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:46.307 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:46.324 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:46.324 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:46.325 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:46.325 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:46.325 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:46.325 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:46.325 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:46.325 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:46.326 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:46.384 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:46.385 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:46.385 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:46.385 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:46.385 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:46.385 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:46.385 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:46.385 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:46.386 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:46.387 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=182), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [39ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [63ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [181ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [181ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [182ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [183ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, deferred=Deferred@379119581(state=PENDING, result=null, callback=(continuation of Deferred@1493081059 after retry RPC after error@1986788537) -> (continuation of Deferred@1883101586 after retry RPC after error@566918727) -> (continuation of Deferred@231200798 after retry RPC after error@610296619) -> (continuation of Deferred@974118780 after retry RPC after error@1212528188) -> (continuation of Deferred@2071035906 after retry RPC after error@1546640764) -> (continuation of Deferred@404688314 after retry RPC after error@2098113747) -> (continuation of Deferred@168317664 after retry RPC after error@678345440), errback=(continuation of Deferred@1493081059 after retry RPC after error@1986788537) -> (continuation of Deferred@1883101586 after retry RPC after error@566918727) -> (continuation of Deferred@231200798 after retry RPC after error@610296619) -> (continuation of Deferred@974118780 after retry RPC after error@1212528188) -> (continuation of Deferred@2071035906 after retry RPC after error@1546640764) -> (continuation of Deferred@404688314 after retry RPC after error@2098113747) -> (continuation of Deferred@168317664 after retry RPC after error@678345440)))
21:04:46.387 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=182), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [39ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [63ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [181ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [181ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [182ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [183ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, deferred=Deferred@379119581(state=PENDING, result=null, callback=(continuation of Deferred@1493081059 after retry RPC after error@1986788537) -> (continuation of Deferred@1883101586 after retry RPC after error@566918727) -> (continuation of Deferred@231200798 after retry RPC after error@610296619) -> (continuation of Deferred@974118780 after retry RPC after error@1212528188) -> (continuation of Deferred@2071035906 after retry RPC after error@1546640764) -> (continuation of Deferred@404688314 after retry RPC after error@2098113747) -> (continuation of Deferred@168317664 after retry RPC after error@678345440), errback=(continuation of Deferred@1493081059 after retry RPC after error@1986788537) -> (continuation of Deferred@1883101586 after retry RPC after error@566918727) -> (continuation of Deferred@231200798 after retry RPC after error@610296619) -> (continuation of Deferred@974118780 after retry RPC after error@1212528188) -> (continuation of Deferred@2071035906 after retry RPC after error@1546640764) -> (continuation of Deferred@404688314 after retry RPC after error@2098113747) -> (continuation of Deferred@168317664 after retry RPC after error@678345440)))
21:04:46.404 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:46.404 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:46.405 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:46.405 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:46.405 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:46.405 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:46.405 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:46.405 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:46.406 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:46.424 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:46.424 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:46.425 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:46.425 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:46.425 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:46.425 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:46.425 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:46.425 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:46.426 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:46.444 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:46.444 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:46.444 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:46.444 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:46.444 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:46.445 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:46.445 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:46.445 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:46.446 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:46.464 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:46.465 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:46.465 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:46.465 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:46.465 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:46.465 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:46.465 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:46.465 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:46.466 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:46.484 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:46.484 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:46.485 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:46.485 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:46.485 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:46.485 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:46.485 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:46.485 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:46.486 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:46.504 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:46.504 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:46.505 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:46.505 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:46.505 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:46.505 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:46.505 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:46.505 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:46.506 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:46.524 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:46.524 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:46.525 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:46.525 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:46.525 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:46.525 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:46.525 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:46.525 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:46.526 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:46.584 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:46.584 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:46.585 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:46.585 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:46.585 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:46.585 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:46.585 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:46.585 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:46.586 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:46.664 [DEBUG - pool-40-thread-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=259), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [39ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [40ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [181ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [181ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [182ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, deferred=Deferred@1280561397(state=PENDING, result=null, callback=(continuation of Deferred@447229398 after retry RPC after error@1856453507) -> (continuation of Deferred@326831587 after retry RPC after error@1098752769) -> (continuation of Deferred@442556517 after retry RPC after error@1296874377) -> (continuation of Deferred@110343126 after retry RPC after error@718957998) -> (continuation of Deferred@933548888 after retry RPC after error@521971414) -> (continuation of Deferred@67812320 after retry RPC after error@2109997522) -> (continuation of Deferred@1818964769 after retry RPC after error@372965996) -> (continuation of Deferred@1195339504 after retry RPC after error@438840660), errback=(continuation of Deferred@447229398 after retry RPC after error@1856453507) -> (continuation of Deferred@326831587 after retry RPC after error@1098752769) -> (continuation of Deferred@442556517 after retry RPC after error@1296874377) -> (continuation of Deferred@110343126 after retry RPC after error@718957998) -> (continuation of Deferred@933548888 after retry RPC after error@521971414) -> (continuation of Deferred@67812320 after retry RPC after error@2109997522) -> (continuation of Deferred@1818964769 after retry RPC after error@372965996) -> (continuation of Deferred@1195339504 after retry RPC after error@438840660)))
21:04:46.665 [WARN - pool-40-thread-1] (HashedWheelTimer.java:716) An exception was thrown by TimerTask.
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343) ~[main/:?]
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at java.lang.Thread.run(Thread.java:748) [?:1.8.0_275]
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86)
at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106)
at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198)
at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158)
at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953)
at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923)
at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464)
at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713)
at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701)
at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788)
at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501)
at java.lang.Thread.run(Thread.java:748)
21:04:46.684 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:46.684 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:46.685 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:46.685 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:46.685 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:46.685 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:46.685 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:46.685 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:46.686 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:46.704 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:46.705 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:46.705 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:46.705 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:46.705 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:46.705 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:46.706 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:46.706 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:46.706 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:46.724 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:46.724 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:46.725 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:46.725 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:46.725 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:46.725 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:46.725 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:46.725 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:46.726 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:46.744 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:46.744 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:46.745 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:46.745 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:46.745 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:46.745 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:46.745 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:46.745 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:46.746 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:46.764 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:46.764 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:46.765 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:46.765 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:46.765 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:46.765 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:46.765 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:46.765 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:46.766 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:46.804 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:46.804 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:46.805 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:46.805 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:46.805 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:46.805 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:46.805 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:46.805 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:46.806 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:46.844 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:46.845 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:46.845 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:46.845 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:46.845 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:46.846 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:46.846 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:46.846 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:46.847 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:46.865 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:46.865 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:46.865 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:46.865 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:46.866 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:46.866 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:46.866 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:46.866 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:46.867 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:46.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:46.898716 24442 raft_consensus.cc:493] T 00000000000000000000000000000000 P eb664add87f44013b596f4cc01c9a41c [term 4 FOLLOWER]: Starting pre-election (detected failure of leader c520077acf3341e29447c6c2b8d3f74c)
21:04:46.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:46.898959 24442 raft_consensus.cc:515] T 00000000000000000000000000000000 P eb664add87f44013b596f4cc01c9a41c [term 4 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e9dd218ca2244f5a934aac2b7e58eb1d" member_type: VOTER last_known_addr { host: "127.22.168.62" port: 37239 } } peers { permanent_uuid: "c520077acf3341e29447c6c2b8d3f74c" member_type: VOTER last_known_addr { host: "127.22.168.61" port: 32947 } } peers { permanent_uuid: "eb664add87f44013b596f4cc01c9a41c" member_type: VOTER last_known_addr { host: "127.22.168.60" port: 33247 } }
21:04:46.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:46.899711 24442 leader_election.cc:290] T 00000000000000000000000000000000 P eb664add87f44013b596f4cc01c9a41c [CANDIDATE]: Term 5 pre-election: Requested pre-vote from peers e9dd218ca2244f5a934aac2b7e58eb1d (127.22.168.62:37239), c520077acf3341e29447c6c2b8d3f74c (127.22.168.61:32947)
21:04:46.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:46.902202 24089 leader_election.cc:336] T 00000000000000000000000000000000 P eb664add87f44013b596f4cc01c9a41c [CANDIDATE]: Term 5 pre-election: RPC error from VoteRequest() call to peer c520077acf3341e29447c6c2b8d3f74c (127.22.168.61:32947): Network error: Client connection negotiation failed: client connection to 127.22.168.61:32947: connect: Connection refused (error 111)
21:04:46.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:46.902433 24086 leader_election.cc:336] T 00000000000000000000000000000000 P eb664add87f44013b596f4cc01c9a41c [CANDIDATE]: Term 5 pre-election: RPC error from VoteRequest() call to peer e9dd218ca2244f5a934aac2b7e58eb1d (127.22.168.62:37239): Network error: Client connection negotiation failed: client connection to 127.22.168.62:37239: connect: Connection refused (error 111)
21:04:46.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:46.902613 24086 leader_election.cc:304] T 00000000000000000000000000000000 P eb664add87f44013b596f4cc01c9a41c [CANDIDATE]: Term 5 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: eb664add87f44013b596f4cc01c9a41c; no voters: c520077acf3341e29447c6c2b8d3f74c, e9dd218ca2244f5a934aac2b7e58eb1d
21:04:46.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:46.902895 24442 raft_consensus.cc:2749] T 00000000000000000000000000000000 P eb664add87f44013b596f4cc01c9a41c [term 4 FOLLOWER]: Leader pre-election lost for term 5. Reason: could not achieve majority
21:04:46.944 [DEBUG - pool-40-thread-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=260), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [161ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [162ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [163ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [163ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [182ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [182ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [183ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [183ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, deferred=Deferred@1255308836(state=PENDING, result=null, callback=(continuation of Deferred@443871822 after retry RPC after error@1297309283) -> (continuation of Deferred@735222046 after retry RPC after error@1915285790) -> (continuation of Deferred@1332976535 after retry RPC after error@411329628) -> (continuation of Deferred@664863803 after retry RPC after error@305443280) -> (continuation of Deferred@1103638306 after retry RPC after error@2060785091) -> (continuation of Deferred@1442465406 after retry RPC after error@1652021024) -> (continuation of Deferred@1648772467 after retry RPC after error@820985845) -> (continuation of Deferred@924931616 after retry RPC after error@1806915891), errback=(continuation of Deferred@443871822 after retry RPC after error@1297309283) -> (continuation of Deferred@735222046 after retry RPC after error@1915285790) -> (continuation of Deferred@1332976535 after retry RPC after error@411329628) -> (continuation of Deferred@664863803 after retry RPC after error@305443280) -> (continuation of Deferred@1103638306 after retry RPC after error@2060785091) -> (continuation of Deferred@1442465406 after retry RPC after error@1652021024) -> (continuation of Deferred@1648772467 after retry RPC after error@820985845) -> (continuation of Deferred@924931616 after retry RPC after error@1806915891)))
21:04:46.945 [WARN - pool-40-thread-1] (HashedWheelTimer.java:716) An exception was thrown by TimerTask.
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343) ~[main/:?]
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at java.lang.Thread.run(Thread.java:748) [?:1.8.0_275]
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86)
at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106)
at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198)
at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158)
at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953)
at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923)
at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464)
at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713)
at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701)
at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788)
at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501)
at java.lang.Thread.run(Thread.java:748)
21:04:46.964 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:46.964 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:46.965 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:46.965 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:46.965 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:46.965 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:46.965 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:46.965 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:46.966 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:46.984 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:46.985 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:46.985 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:46.985 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:46.985 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:46.985 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:46.986 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:46.986 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:46.987 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:47.004 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:47.005 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:47.005 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:47.005 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:47.005 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:47.005 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:47.005 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:47.006 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:47.006 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:47.024 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:47.024 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:47.025 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:47.025 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:47.025 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:47.025 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:47.025 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:47.025 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:47.026 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:47.044 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:47.044 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:47.045 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:47.045 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:47.045 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:47.045 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:47.045 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:47.045 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:47.046 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:47.084 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:47.084 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:47.085 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:47.085 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:47.085 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:47.085 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:47.085 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:47.085 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:47.086 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:47.124 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:47.124 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:47.124 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:47.125 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:47.125 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:47.124 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:47.125 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:47.125 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:47.126 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:47.165 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:47.165 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:47.165 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:47.165 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:47.166 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:47.166 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:47.166 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:47.166 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:47.166 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:47.167 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=203), Traces: [0ms] refreshing cache from master, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [3ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [21ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [22ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [22ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [23ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [23ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [23ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [24ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [41ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [43ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [61ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [63ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [81ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [83ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [121ms] refreshing cache from master, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [162ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [162ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [163ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [163ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [201ms] refreshing cache from master, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [202ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [202ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [203ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [203ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [203ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [203ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, deferred=Deferred@436222403(state=PENDING, result=null, callback=(continuation of Deferred@1901554034 after retry RPC after error@265156801) -> (continuation of Deferred@150578697 after retry RPC after error@321673460) -> (continuation of Deferred@1501212442 after retry RPC after error@957435652) -> (continuation of Deferred@2090696641 after retry RPC after error@226980702) -> (continuation of Deferred@25759627 after retry RPC after error@852691409) -> (continuation of Deferred@631708477 after retry RPC after error@327975252) -> (continuation of Deferred@1513103865 after retry RPC after error@1879521007), errback=(continuation of Deferred@1901554034 after retry RPC after error@265156801) -> (continuation of Deferred@150578697 after retry RPC after error@321673460) -> (continuation of Deferred@1501212442 after retry RPC after error@957435652) -> (continuation of Deferred@2090696641 after retry RPC after error@226980702) -> (continuation of Deferred@25759627 after retry RPC after error@852691409) -> (continuation of Deferred@631708477 after retry RPC after error@327975252) -> (continuation of Deferred@1513103865 after retry RPC after error@1879521007)))
21:04:47.167 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=203), Traces: [0ms] refreshing cache from master, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [3ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [21ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [22ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [22ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [23ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [23ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [23ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [24ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [41ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [43ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [61ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [63ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [81ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [83ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [121ms] refreshing cache from master, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [162ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [162ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [163ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [163ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [201ms] refreshing cache from master, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [202ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [202ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [203ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [203ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [203ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [203ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, deferred=Deferred@436222403(state=PENDING, result=null, callback=(continuation of Deferred@1901554034 after retry RPC after error@265156801) -> (continuation of Deferred@150578697 after retry RPC after error@321673460) -> (continuation of Deferred@1501212442 after retry RPC after error@957435652) -> (continuation of Deferred@2090696641 after retry RPC after error@226980702) -> (continuation of Deferred@25759627 after retry RPC after error@852691409) -> (continuation of Deferred@631708477 after retry RPC after error@327975252) -> (continuation of Deferred@1513103865 after retry RPC after error@1879521007), errback=(continuation of Deferred@1901554034 after retry RPC after error@265156801) -> (continuation of Deferred@150578697 after retry RPC after error@321673460) -> (continuation of Deferred@1501212442 after retry RPC after error@957435652) -> (continuation of Deferred@2090696641 after retry RPC after error@226980702) -> (continuation of Deferred@25759627 after retry RPC after error@852691409) -> (continuation of Deferred@631708477 after retry RPC after error@327975252) -> (continuation of Deferred@1513103865 after retry RPC after error@1879521007)))
21:04:47.184 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:47.185 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:47.185 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:47.185 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:47.185 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:47.185 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:47.185 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:47.185 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:47.187 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:47.204 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:47.204 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:47.205 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:47.205 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:47.205 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:47.205 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:47.205 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:47.205 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:47.206 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:47.224 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:47.224 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:47.225 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:47.225 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:47.225 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:47.225 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:47.225 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:47.225 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:47.226 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:47.244 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:47.245 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:47.245 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:47.245 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:47.245 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:47.245 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:47.246 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:47.246 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:47.246 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:47.264 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:47.264 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:47.265 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:47.265 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:47.265 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:47.265 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:47.266 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:47.266 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:47.266 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:47.284 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:47.284 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:47.285 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:47.285 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:47.285 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:47.285 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:47.285 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:47.285 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:47.286 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:47.324 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:47.324 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:47.325 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:47.325 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:47.325 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:47.325 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:47.325 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:47.325 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:47.326 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:47.344 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:47.344 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:47.345 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:47.345 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:47.345 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:47.345 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:47.345 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:47.345 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:47.346 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:47.384 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:47.384 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:47.384 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:47.385 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:47.385 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:47.386 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:47.386 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:47.386 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:47.386 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:47.404 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:47.404 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:47.405 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:47.405 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:47.405 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:47.408 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:47.409 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:47.409 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:47.409 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:47.410 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=9, TimeoutTracker(timeout=250, elapsed=225), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [39ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [141ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [141ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [142ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [199ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [201ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [202ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [202ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [202ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [221ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [225ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [225ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [226ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, deferred=Deferred@327975430(state=PENDING, result=null, callback=(continuation of Deferred@1969608432 after retry RPC after error@331600661) -> (continuation of Deferred@542713855 after retry RPC after error@1471290688) -> (continuation of Deferred@213615376 after retry RPC after error@23962345) -> (continuation of Deferred@1751256036 after retry RPC after error@796742581) -> (continuation of Deferred@2131682889 after retry RPC after error@328759723) -> (continuation of Deferred@1801288983 after retry RPC after error@788032306) -> (continuation of Deferred@650968074 after retry RPC after error@1992920153) -> (continuation of Deferred@1996288594 after retry RPC after error@464565828), errback=(continuation of Deferred@1969608432 after retry RPC after error@331600661) -> (continuation of Deferred@542713855 after retry RPC after error@1471290688) -> (continuation of Deferred@213615376 after retry RPC after error@23962345) -> (continuation of Deferred@1751256036 after retry RPC after error@796742581) -> (continuation of Deferred@2131682889 after retry RPC after error@328759723) -> (continuation of Deferred@1801288983 after retry RPC after error@788032306) -> (continuation of Deferred@650968074 after retry RPC after error@1992920153) -> (continuation of Deferred@1996288594 after retry RPC after error@464565828)))
21:04:47.411 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=9, TimeoutTracker(timeout=250, elapsed=225), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [39ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [141ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [141ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [142ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [199ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [201ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [202ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [202ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [202ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [221ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [225ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [225ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [226ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, deferred=Deferred@327975430(state=PENDING, result=null, callback=(continuation of Deferred@1969608432 after retry RPC after error@331600661) -> (continuation of Deferred@542713855 after retry RPC after error@1471290688) -> (continuation of Deferred@213615376 after retry RPC after error@23962345) -> (continuation of Deferred@1751256036 after retry RPC after error@796742581) -> (continuation of Deferred@2131682889 after retry RPC after error@328759723) -> (continuation of Deferred@1801288983 after retry RPC after error@788032306) -> (continuation of Deferred@650968074 after retry RPC after error@1992920153) -> (continuation of Deferred@1996288594 after retry RPC after error@464565828), errback=(continuation of Deferred@1969608432 after retry RPC after error@331600661) -> (continuation of Deferred@542713855 after retry RPC after error@1471290688) -> (continuation of Deferred@213615376 after retry RPC after error@23962345) -> (continuation of Deferred@1751256036 after retry RPC after error@796742581) -> (continuation of Deferred@2131682889 after retry RPC after error@328759723) -> (continuation of Deferred@1801288983 after retry RPC after error@788032306) -> (continuation of Deferred@650968074 after retry RPC after error@1992920153) -> (continuation of Deferred@1996288594 after retry RPC after error@464565828)))
21:04:47.424 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:47.424 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:47.425 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:47.425 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:47.425 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:47.425 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:47.425 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:47.425 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:47.426 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:47.444 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:47.444 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:47.444 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:47.445 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:47.445 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:47.445 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:47.445 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:47.445 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:47.446 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:47.464 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:47.465 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:47.465 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:47.465 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:47.465 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:47.466 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:47.466 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:47.466 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:47.466 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:47.484 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:47.484 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:47.485 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:47.485 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:47.485 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:47.485 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:47.485 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:47.485 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:47.486 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:47.504 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:47.504 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:47.505 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:47.505 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:47.505 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:47.505 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:47.505 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:47.505 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:47.506 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:47.524 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:47.524 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:47.525 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:47.525 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:47.525 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:47.525 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:47.525 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:47.525 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:47.526 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:47.545 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:47.545 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:47.546 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:47.546 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:47.546 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:47.546 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:47.546 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:47.546 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:47.547 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:47.664 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:47.665 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:47.665 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:47.665 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:47.665 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:47.665 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:47.665 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:47.665 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:47.667 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:47.667 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=243), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [19ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [99ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [123ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [241ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [241ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [242ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [243ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, deferred=Deferred@1587484513(state=PENDING, result=null, callback=(continuation of Deferred@1737242208 after retry RPC after error@1539623665) -> (continuation of Deferred@380531259 after retry RPC after error@808213392) -> (continuation of Deferred@915756227 after retry RPC after error@1274778530) -> (continuation of Deferred@773918540 after retry RPC after error@2088972739) -> (continuation of Deferred@1621905884 after retry RPC after error@2002626792) -> (continuation of Deferred@1369477664 after retry RPC after error@877178110) -> (continuation of Deferred@921188634 after retry RPC after error@745070023), errback=(continuation of Deferred@1737242208 after retry RPC after error@1539623665) -> (continuation of Deferred@380531259 after retry RPC after error@808213392) -> (continuation of Deferred@915756227 after retry RPC after error@1274778530) -> (continuation of Deferred@773918540 after retry RPC after error@2088972739) -> (continuation of Deferred@1621905884 after retry RPC after error@2002626792) -> (continuation of Deferred@1369477664 after retry RPC after error@877178110) -> (continuation of Deferred@921188634 after retry RPC after error@745070023)))
21:04:47.667 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=243), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [19ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [99ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [123ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [241ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [241ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [242ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [243ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, deferred=Deferred@1587484513(state=PENDING, result=null, callback=(continuation of Deferred@1737242208 after retry RPC after error@1539623665) -> (continuation of Deferred@380531259 after retry RPC after error@808213392) -> (continuation of Deferred@915756227 after retry RPC after error@1274778530) -> (continuation of Deferred@773918540 after retry RPC after error@2088972739) -> (continuation of Deferred@1621905884 after retry RPC after error@2002626792) -> (continuation of Deferred@1369477664 after retry RPC after error@877178110) -> (continuation of Deferred@921188634 after retry RPC after error@745070023), errback=(continuation of Deferred@1737242208 after retry RPC after error@1539623665) -> (continuation of Deferred@380531259 after retry RPC after error@808213392) -> (continuation of Deferred@915756227 after retry RPC after error@1274778530) -> (continuation of Deferred@773918540 after retry RPC after error@2088972739) -> (continuation of Deferred@1621905884 after retry RPC after error@2002626792) -> (continuation of Deferred@1369477664 after retry RPC after error@877178110) -> (continuation of Deferred@921188634 after retry RPC after error@745070023)))
21:04:47.684 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:47.684 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:47.685 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:47.685 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:47.685 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:47.685 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:47.685 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:47.685 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:47.686 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:47.704 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:47.704 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:47.705 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:47.705 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:47.705 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:47.705 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:47.705 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:47.705 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:47.706 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:47.724 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:47.724 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:47.725 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:47.725 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:47.725 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:47.725 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:47.725 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:47.725 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:47.726 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:47.745 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:47.745 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:47.746 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:47.746 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:47.746 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:47.746 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:47.746 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:47.746 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:47.747 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:47.765 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:47.765 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:47.766 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:47.766 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:47.766 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:47.766 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:47.766 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:47.766 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:47.767 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:47.805 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:47.805 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:47.805 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:47.805 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:47.805 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:47.806 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:47.806 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:47.806 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:47.806 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:47.824 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:47.825 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:47.825 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:47.825 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:47.825 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:47.825 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:47.825 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:47.825 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:47.827 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:47.884 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:47.885 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:47.885 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:47.885 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:47.885 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:47.885 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:47.885 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:47.885 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:47.887 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:47.887 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=202), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [39ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [60ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [63ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [83ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [141ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [141ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [143ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [143ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [201ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [201ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [202ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [203ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, deferred=Deferred@1824421097(state=PENDING, result=null, callback=(continuation of Deferred@325454511 after retry RPC after error@1647956569) -> (continuation of Deferred@1954199829 after retry RPC after error@1803326031) -> (continuation of Deferred@1215094525 after retry RPC after error@1693148851) -> (continuation of Deferred@1173932330 after retry RPC after error@1251595568) -> (continuation of Deferred@1680634280 after retry RPC after error@1540185440) -> (continuation of Deferred@1484750506 after retry RPC after error@1556811968) -> (continuation of Deferred@1533909952 after retry RPC after error@1786444688), errback=(continuation of Deferred@325454511 after retry RPC after error@1647956569) -> (continuation of Deferred@1954199829 after retry RPC after error@1803326031) -> (continuation of Deferred@1215094525 after retry RPC after error@1693148851) -> (continuation of Deferred@1173932330 after retry RPC after error@1251595568) -> (continuation of Deferred@1680634280 after retry RPC after error@1540185440) -> (continuation of Deferred@1484750506 after retry RPC after error@1556811968) -> (continuation of Deferred@1533909952 after retry RPC after error@1786444688)))
21:04:47.888 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=202), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [39ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [60ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [63ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [83ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [141ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [141ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [143ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [143ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [201ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [201ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [202ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [203ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, deferred=Deferred@1824421097(state=PENDING, result=null, callback=(continuation of Deferred@325454511 after retry RPC after error@1647956569) -> (continuation of Deferred@1954199829 after retry RPC after error@1803326031) -> (continuation of Deferred@1215094525 after retry RPC after error@1693148851) -> (continuation of Deferred@1173932330 after retry RPC after error@1251595568) -> (continuation of Deferred@1680634280 after retry RPC after error@1540185440) -> (continuation of Deferred@1484750506 after retry RPC after error@1556811968) -> (continuation of Deferred@1533909952 after retry RPC after error@1786444688), errback=(continuation of Deferred@325454511 after retry RPC after error@1647956569) -> (continuation of Deferred@1954199829 after retry RPC after error@1803326031) -> (continuation of Deferred@1215094525 after retry RPC after error@1693148851) -> (continuation of Deferred@1173932330 after retry RPC after error@1251595568) -> (continuation of Deferred@1680634280 after retry RPC after error@1540185440) -> (continuation of Deferred@1484750506 after retry RPC after error@1556811968) -> (continuation of Deferred@1533909952 after retry RPC after error@1786444688)))
21:04:47.904 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:47.904 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:47.905 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:47.905 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:47.906 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:47.906 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:47.906 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:47.906 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:47.907 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:47.924 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:47.924 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:47.925 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:47.925 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:47.925 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:47.925 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:47.925 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:47.925 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:47.926 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:47.944 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:47.944 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:47.945 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:47.945 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:47.945 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:47.945 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:47.945 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:47.945 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:47.946 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:47.965 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:47.965 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:47.965 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:47.965 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:47.966 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:47.966 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:47.966 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:47.966 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:47.967 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:47.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:47.975894 23569 master_proxy_rpc.cc:177] Failed to determine new leader Master: Timed out: ConnectToClusterRpc(addrs: 127.22.168.62:37239,127.22.168.61:32947,127.22.168.60:33247, num_attempts: 97) passed its deadline: Not found: no leader found: ConnectToClusterRpc(addrs: 127.22.168.62:37239,127.22.168.61:32947,127.22.168.60:33247, num_attempts: 1)
21:04:47.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:47.981388 23569 meta_cache.cc:788] Timed out: LookupRpcById { tablet: '01aa0e192c434b468ec7ed90d48563b3', attempt: 2 } failed: LookupRpcById timed out after deadline expired: LookupRpcById { tablet: '01aa0e192c434b468ec7ed90d48563b3', attempt: 2 } passed its deadline: Timed out: ConnectToClusterRpc(addrs: 127.22.168.62:37239,127.22.168.61:32947,127.22.168.60:33247, num_attempts: 97) passed its deadline: Not found: no leader found: ConnectToClusterRpc(addrs: 127.22.168.62:37239,127.22.168.61:32947,127.22.168.60:33247, num_attempts: 1)
21:04:47.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:47.983603 23569 master_proxy_rpc.cc:203] Re-attempting LookupRpcById request to leader Master (127.22.168.60:33247)
21:04:47.984 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:47.985 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:47.985 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:47.985 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:47.985 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:47.985 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:47.985 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:47.985 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:47.986 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:48.004 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:48.004 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:48.005 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:48.005 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:48.005 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:48.005 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:48.005 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:48.005 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:48.006 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:48.024 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:48.024 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:48.025 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:48.025 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:48.025 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:48.025 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:48.025 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:48.025 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:48.026 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:48.104 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:48.104 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:48.105 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:48.105 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:48.105 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:48.105 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:48.105 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:48.105 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:48.106 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:48.144 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:48.144 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:48.145 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:48.145 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:48.145 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:48.145 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:48.145 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:48.145 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:48.146 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:48.147 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=9, TimeoutTracker(timeout=250, elapsed=242), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [3ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [63ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [201ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [201ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [202ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [241ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [241ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [242ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [242ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, deferred=Deferred@1458816933(state=PENDING, result=null, callback=(continuation of Deferred@1502190809 after retry RPC after error@1501906577) -> (continuation of Deferred@1394492661 after retry RPC after error@462979036) -> (continuation of Deferred@249714466 after retry RPC after error@2099263905) -> (continuation of Deferred@1547236124 after retry RPC after error@67506648) -> (continuation of Deferred@415309117 after retry RPC after error@1691402933) -> (continuation of Deferred@2137746720 after retry RPC after error@1264004036) -> (continuation of Deferred@1534066534 after retry RPC after error@969432020) -> (continuation of Deferred@493761440 after retry RPC after error@1872044738), errback=(continuation of Deferred@1502190809 after retry RPC after error@1501906577) -> (continuation of Deferred@1394492661 after retry RPC after error@462979036) -> (continuation of Deferred@249714466 after retry RPC after error@2099263905) -> (continuation of Deferred@1547236124 after retry RPC after error@67506648) -> (continuation of Deferred@415309117 after retry RPC after error@1691402933) -> (continuation of Deferred@2137746720 after retry RPC after error@1264004036) -> (continuation of Deferred@1534066534 after retry RPC after error@969432020) -> (continuation of Deferred@493761440 after retry RPC after error@1872044738)))
21:04:48.147 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=9, TimeoutTracker(timeout=250, elapsed=242), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [3ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [63ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [201ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [201ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [202ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [241ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [241ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [242ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [242ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, deferred=Deferred@1458816933(state=PENDING, result=null, callback=(continuation of Deferred@1502190809 after retry RPC after error@1501906577) -> (continuation of Deferred@1394492661 after retry RPC after error@462979036) -> (continuation of Deferred@249714466 after retry RPC after error@2099263905) -> (continuation of Deferred@1547236124 after retry RPC after error@67506648) -> (continuation of Deferred@415309117 after retry RPC after error@1691402933) -> (continuation of Deferred@2137746720 after retry RPC after error@1264004036) -> (continuation of Deferred@1534066534 after retry RPC after error@969432020) -> (continuation of Deferred@493761440 after retry RPC after error@1872044738), errback=(continuation of Deferred@1502190809 after retry RPC after error@1501906577) -> (continuation of Deferred@1394492661 after retry RPC after error@462979036) -> (continuation of Deferred@249714466 after retry RPC after error@2099263905) -> (continuation of Deferred@1547236124 after retry RPC after error@67506648) -> (continuation of Deferred@415309117 after retry RPC after error@1691402933) -> (continuation of Deferred@2137746720 after retry RPC after error@1264004036) -> (continuation of Deferred@1534066534 after retry RPC after error@969432020) -> (continuation of Deferred@493761440 after retry RPC after error@1872044738)))
21:04:48.164 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:48.164 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:48.165 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:48.165 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:48.165 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:48.165 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:48.165 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:48.165 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:48.166 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:48.184 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:48.184 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:48.185 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:48.185 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:48.185 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:48.185 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:48.185 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:48.185 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:48.186 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:48.205 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:48.205 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:48.205 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:48.205 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:48.205 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:48.205 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:48.206 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:48.206 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:48.207 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:48.224 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:48.225 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:48.225 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:48.225 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:48.225 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:48.225 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:48.225 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:48.225 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:48.226 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:48.244 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:48.244 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:48.245 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:48.245 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:48.245 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:48.245 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:48.245 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:48.245 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:48.246 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:48.284 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:48.284 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:48.285 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:48.285 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:48.285 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:48.285 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:48.285 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:48.285 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:48.286 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:48.324 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:48.324 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:48.325 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:48.325 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:48.325 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:48.325 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:48.325 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:48.325 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:48.326 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:48.424 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:48.424 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:48.425 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:48.425 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:48.425 [DEBUG - pool-40-thread-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=260), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [161ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [161ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [162ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, deferred=Deferred@1571688708(state=PENDING, result=null, callback=(continuation of Deferred@573076863 after retry RPC after error@1538990055) -> (continuation of Deferred@750418693 after retry RPC after error@20019623) -> (continuation of Deferred@1616082803 after retry RPC after error@1670645945) -> (continuation of Deferred@478754205 after retry RPC after error@385846250) -> (continuation of Deferred@1835835049 after retry RPC after error@780046973) -> (continuation of Deferred@2064543730 after retry RPC after error@298064802) -> (continuation of Deferred@416569061 after retry RPC after error@1122226936), errback=(continuation of Deferred@573076863 after retry RPC after error@1538990055) -> (continuation of Deferred@750418693 after retry RPC after error@20019623) -> (continuation of Deferred@1616082803 after retry RPC after error@1670645945) -> (continuation of Deferred@478754205 after retry RPC after error@385846250) -> (continuation of Deferred@1835835049 after retry RPC after error@780046973) -> (continuation of Deferred@2064543730 after retry RPC after error@298064802) -> (continuation of Deferred@416569061 after retry RPC after error@1122226936)))
21:04:48.426 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:48.426 [WARN - pool-40-thread-1] (HashedWheelTimer.java:716) An exception was thrown by TimerTask.
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343) ~[main/:?]
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at java.lang.Thread.run(Thread.java:748) [?:1.8.0_275]
21:04:48.425 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86)
at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106)
at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198)
at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158)
at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953)
at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923)
at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464)
at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713)
at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701)
at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788)
at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501)
at java.lang.Thread.run(Thread.java:748)
21:04:48.426 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:48.427 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:48.427 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:48.444 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:48.444 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:48.445 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:48.445 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:48.445 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:48.445 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:48.445 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:48.445 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:48.446 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:48.464 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:48.464 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:48.464 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:48.464 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:48.465 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:48.465 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:48.465 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:48.465 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:48.466 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:48.484 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:48.485 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:48.485 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:48.485 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:48.485 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:48.485 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:48.485 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:48.485 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:48.486 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:48.504 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:48.504 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:48.505 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:48.505 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:48.505 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:48.505 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:48.505 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:48.505 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:48.506 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:48.524 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:48.524 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:48.525 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:48.525 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:48.525 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:48.525 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:48.525 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:48.525 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:48.526 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:48.564 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:48.564 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:48.565 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:48.565 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:48.565 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:48.565 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:48.565 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:48.565 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:48.566 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:48.604 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:48.604 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:48.605 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:48.605 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:48.605 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:48.605 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:48.605 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:48.605 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:48.606 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:48.664 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:48.664 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:48.665 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:48.665 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:48.665 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:48.665 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:48.665 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:48.665 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:48.666 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:48.667 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=222), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [19ms] refreshing cache from master, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [161ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [161ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [162ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [221ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [221ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [222ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [222ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, deferred=Deferred@1508383677(state=PENDING, result=null, callback=(continuation of Deferred@1951461761 after retry RPC after error@2127171886) -> (continuation of Deferred@1516500191 after retry RPC after error@1129261533) -> (continuation of Deferred@811603148 after retry RPC after error@512569487) -> (continuation of Deferred@1536027277 after retry RPC after error@1630937568) -> (continuation of Deferred@1371574142 after retry RPC after error@641651841) -> (continuation of Deferred@315853641 after retry RPC after error@1119523282) -> (continuation of Deferred@399793540 after retry RPC after error@1201539363), errback=(continuation of Deferred@1951461761 after retry RPC after error@2127171886) -> (continuation of Deferred@1516500191 after retry RPC after error@1129261533) -> (continuation of Deferred@811603148 after retry RPC after error@512569487) -> (continuation of Deferred@1536027277 after retry RPC after error@1630937568) -> (continuation of Deferred@1371574142 after retry RPC after error@641651841) -> (continuation of Deferred@315853641 after retry RPC after error@1119523282) -> (continuation of Deferred@399793540 after retry RPC after error@1201539363)))
21:04:48.667 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=222), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [19ms] refreshing cache from master, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [161ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [161ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [162ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [221ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [221ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [222ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [222ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, deferred=Deferred@1508383677(state=PENDING, result=null, callback=(continuation of Deferred@1951461761 after retry RPC after error@2127171886) -> (continuation of Deferred@1516500191 after retry RPC after error@1129261533) -> (continuation of Deferred@811603148 after retry RPC after error@512569487) -> (continuation of Deferred@1536027277 after retry RPC after error@1630937568) -> (continuation of Deferred@1371574142 after retry RPC after error@641651841) -> (continuation of Deferred@315853641 after retry RPC after error@1119523282) -> (continuation of Deferred@399793540 after retry RPC after error@1201539363), errback=(continuation of Deferred@1951461761 after retry RPC after error@2127171886) -> (continuation of Deferred@1516500191 after retry RPC after error@1129261533) -> (continuation of Deferred@811603148 after retry RPC after error@512569487) -> (continuation of Deferred@1536027277 after retry RPC after error@1630937568) -> (continuation of Deferred@1371574142 after retry RPC after error@641651841) -> (continuation of Deferred@315853641 after retry RPC after error@1119523282) -> (continuation of Deferred@399793540 after retry RPC after error@1201539363)))
21:04:48.684 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:48.684 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:48.685 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:48.685 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:48.685 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:48.685 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:48.685 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:48.685 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:48.686 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:48.704 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:48.704 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:48.704 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:48.704 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:48.705 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:48.705 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:48.705 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:48.705 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:48.706 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:48.724 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:48.725 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:48.725 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:48.725 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:48.726 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:48.726 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:48.726 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:48.726 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:48.727 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:48.744 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:48.744 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:48.745 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:48.745 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:48.745 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:48.745 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:48.745 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:48.745 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:48.746 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:48.764 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:48.764 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:48.765 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:48.765 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:48.765 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:48.765 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:48.765 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:48.765 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:48.766 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:48.804 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:48.804 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:48.805 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:48.805 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:48.805 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:48.805 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:48.805 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:48.805 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:48.807 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:48.844 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:48.844 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:48.845 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:48.845 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:48.845 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:48.845 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:48.845 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:48.845 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:48.846 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:48.864 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:48.865 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:48.865 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:48.865 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:48.865 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:48.865 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:48.865 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:48.865 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:48.866 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:48.867 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=182), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [19ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [43ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [123ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [161ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [161ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [162ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [181ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [181ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [182ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, deferred=Deferred@362406609(state=PENDING, result=null, callback=(continuation of Deferred@702058170 after retry RPC after error@206809608) -> (continuation of Deferred@1834516488 after retry RPC after error@932018200) -> (continuation of Deferred@1523424525 after retry RPC after error@1272938422) -> (continuation of Deferred@1655631365 after retry RPC after error@1044322288) -> (continuation of Deferred@137154767 after retry RPC after error@951987379) -> (continuation of Deferred@762655591 after retry RPC after error@1806901678) -> (continuation of Deferred@100085439 after retry RPC after error@1290096902), errback=(continuation of Deferred@702058170 after retry RPC after error@206809608) -> (continuation of Deferred@1834516488 after retry RPC after error@932018200) -> (continuation of Deferred@1523424525 after retry RPC after error@1272938422) -> (continuation of Deferred@1655631365 after retry RPC after error@1044322288) -> (continuation of Deferred@137154767 after retry RPC after error@951987379) -> (continuation of Deferred@762655591 after retry RPC after error@1806901678) -> (continuation of Deferred@100085439 after retry RPC after error@1290096902)))
21:04:48.867 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=182), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [19ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [43ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [123ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [161ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [161ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [162ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [181ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [181ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [182ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, deferred=Deferred@362406609(state=PENDING, result=null, callback=(continuation of Deferred@702058170 after retry RPC after error@206809608) -> (continuation of Deferred@1834516488 after retry RPC after error@932018200) -> (continuation of Deferred@1523424525 after retry RPC after error@1272938422) -> (continuation of Deferred@1655631365 after retry RPC after error@1044322288) -> (continuation of Deferred@137154767 after retry RPC after error@951987379) -> (continuation of Deferred@762655591 after retry RPC after error@1806901678) -> (continuation of Deferred@100085439 after retry RPC after error@1290096902), errback=(continuation of Deferred@702058170 after retry RPC after error@206809608) -> (continuation of Deferred@1834516488 after retry RPC after error@932018200) -> (continuation of Deferred@1523424525 after retry RPC after error@1272938422) -> (continuation of Deferred@1655631365 after retry RPC after error@1044322288) -> (continuation of Deferred@137154767 after retry RPC after error@951987379) -> (continuation of Deferred@762655591 after retry RPC after error@1806901678) -> (continuation of Deferred@100085439 after retry RPC after error@1290096902)))
21:04:48.884 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:48.884 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:48.885 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:48.885 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:48.885 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:48.885 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:48.885 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:48.885 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:48.886 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:48.904 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:48.904 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:48.905 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:48.905 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:48.905 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:48.905 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:48.905 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:48.905 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:48.906 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:48.924 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:48.924 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:48.925 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:48.925 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:48.925 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:48.925 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:48.925 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:48.925 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:48.926 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:48.945 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:48.945 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:48.945 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:48.945 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:48.945 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:48.945 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:48.946 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:48.946 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:48.947 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:48.964 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:48.965 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:48.965 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:48.965 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:48.965 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:48.965 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:48.965 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:48.965 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:48.966 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:49.004 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:49.004 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:49.005 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:49.005 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:49.005 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:49.005 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:49.005 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:49.005 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:49.007 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:49.064 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:49.064 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:49.065 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:49.065 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:49.065 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:49.065 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:49.065 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:49.065 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:49.066 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:49.124 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:49.124 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:49.125 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:49.125 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:49.125 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:49.125 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:49.125 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:49.125 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:49.126 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:49.127 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=242), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [123ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [181ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [181ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [182ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [241ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [241ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [242ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [242ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, deferred=Deferred@568143184(state=PENDING, result=null, callback=(continuation of Deferred@406971219 after retry RPC after error@1081472092) -> (continuation of Deferred@301551283 after retry RPC after error@1501972081) -> (continuation of Deferred@1313500521 after retry RPC after error@561166312) -> (continuation of Deferred@2057693603 after retry RPC after error@293313864) -> (continuation of Deferred@214407979 after retry RPC after error@952021164) -> (continuation of Deferred@1828957929 after retry RPC after error@543158236) -> (continuation of Deferred@284239423 after retry RPC after error@1560792925), errback=(continuation of Deferred@406971219 after retry RPC after error@1081472092) -> (continuation of Deferred@301551283 after retry RPC after error@1501972081) -> (continuation of Deferred@1313500521 after retry RPC after error@561166312) -> (continuation of Deferred@2057693603 after retry RPC after error@293313864) -> (continuation of Deferred@214407979 after retry RPC after error@952021164) -> (continuation of Deferred@1828957929 after retry RPC after error@543158236) -> (continuation of Deferred@284239423 after retry RPC after error@1560792925)))
21:04:49.127 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=242), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [123ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [181ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [181ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [182ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [241ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [241ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [242ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [242ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, deferred=Deferred@568143184(state=PENDING, result=null, callback=(continuation of Deferred@406971219 after retry RPC after error@1081472092) -> (continuation of Deferred@301551283 after retry RPC after error@1501972081) -> (continuation of Deferred@1313500521 after retry RPC after error@561166312) -> (continuation of Deferred@2057693603 after retry RPC after error@293313864) -> (continuation of Deferred@214407979 after retry RPC after error@952021164) -> (continuation of Deferred@1828957929 after retry RPC after error@543158236) -> (continuation of Deferred@284239423 after retry RPC after error@1560792925), errback=(continuation of Deferred@406971219 after retry RPC after error@1081472092) -> (continuation of Deferred@301551283 after retry RPC after error@1501972081) -> (continuation of Deferred@1313500521 after retry RPC after error@561166312) -> (continuation of Deferred@2057693603 after retry RPC after error@293313864) -> (continuation of Deferred@214407979 after retry RPC after error@952021164) -> (continuation of Deferred@1828957929 after retry RPC after error@543158236) -> (continuation of Deferred@284239423 after retry RPC after error@1560792925)))
21:04:49.144 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:49.144 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:49.145 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:49.145 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:49.145 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:49.145 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:49.145 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:49.145 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:49.146 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:49.164 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:49.164 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:49.165 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:49.165 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:49.165 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:49.165 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:49.165 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:49.165 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:49.166 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:49.184 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:49.184 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:49.184 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:49.185 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:49.185 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:49.185 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:49.185 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:49.185 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:49.186 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:49.204 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:49.204 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:49.205 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:49.205 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:49.205 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:49.205 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:49.205 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:49.205 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:49.206 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:49.224 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:49.225 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:49.225 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:49.225 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:49.225 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:49.225 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:49.226 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:49.226 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:49.226 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:49.244 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:49.244 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:49.245 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:49.245 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:49.245 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:49.245 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:49.245 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:49.245 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:49.246 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:49.304 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:49.304 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:49.305 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:49.305 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:49.305 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:49.305 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:49.306 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:49.306 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:49.306 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:49.344 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:49.344 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:49.345 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:49.345 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:49.345 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:49.345 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:49.345 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:49.345 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:49.346 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:49.404 [DEBUG - pool-40-thread-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=260), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [39ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [59ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [161ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [162ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [162ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [162ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [201ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [201ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [202ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, deferred=Deferred@1821619161(state=PENDING, result=null, callback=(continuation of Deferred@1981007258 after retry RPC after error@1095534285) -> (continuation of Deferred@1996931887 after retry RPC after error@117091181) -> (continuation of Deferred@1848475965 after retry RPC after error@1265764571) -> (continuation of Deferred@179090721 after retry RPC after error@1507776218) -> (continuation of Deferred@1534623925 after retry RPC after error@1535953) -> (continuation of Deferred@1870518095 after retry RPC after error@1077589116) -> (continuation of Deferred@1608051174 after retry RPC after error@1678408747) -> (continuation of Deferred@1699485770 after retry RPC after error@1883294155), errback=(continuation of Deferred@1981007258 after retry RPC after error@1095534285) -> (continuation of Deferred@1996931887 after retry RPC after error@117091181) -> (continuation of Deferred@1848475965 after retry RPC after error@1265764571) -> (continuation of Deferred@179090721 after retry RPC after error@1507776218) -> (continuation of Deferred@1534623925 after retry RPC after error@1535953) -> (continuation of Deferred@1870518095 after retry RPC after error@1077589116) -> (continuation of Deferred@1608051174 after retry RPC after error@1678408747) -> (continuation of Deferred@1699485770 after retry RPC after error@1883294155)))
21:04:49.405 [WARN - pool-40-thread-1] (HashedWheelTimer.java:716) An exception was thrown by TimerTask.
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343) ~[main/:?]
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at java.lang.Thread.run(Thread.java:748) [?:1.8.0_275]
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86)
at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106)
at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198)
at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158)
at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953)
at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923)
at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464)
at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713)
at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701)
at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788)
at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501)
at java.lang.Thread.run(Thread.java:748)
21:04:49.424 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:49.424 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:49.425 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:49.425 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:49.425 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:49.425 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:49.425 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:49.425 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:49.426 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:49.445 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:49.445 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:49.445 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:49.445 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:49.446 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:49.446 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:49.446 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:49.446 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:49.446 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:49.464 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:49.465 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:49.465 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:49.465 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:49.465 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:49.465 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:49.465 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:49.465 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:49.466 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:49.484 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:49.484 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:49.485 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:49.485 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:49.485 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:49.485 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:49.485 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:49.485 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:49.486 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:49.504 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:49.504 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:49.505 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:49.505 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:49.505 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:49.505 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:49.505 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:49.505 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:49.506 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:49.544 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:49.544 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:49.545 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:49.545 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:49.545 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:49.545 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:49.545 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:49.545 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:49.546 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:49.564 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:49.564 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:49.565 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:49.565 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:49.565 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:49.565 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:49.565 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:49.565 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:49.566 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:49.685 [DEBUG - pool-40-thread-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=260), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [141ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [141ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [141ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, deferred=Deferred@1336409730(state=PENDING, result=null, callback=(continuation of Deferred@781989335 after retry RPC after error@478791767) -> (continuation of Deferred@586453895 after retry RPC after error@1389836981) -> (continuation of Deferred@489332661 after retry RPC after error@1328752439) -> (continuation of Deferred@1230896637 after retry RPC after error@573863153) -> (continuation of Deferred@1806336053 after retry RPC after error@1039534312) -> (continuation of Deferred@459454654 after retry RPC after error@451986655) -> (continuation of Deferred@635720860 after retry RPC after error@676899200), errback=(continuation of Deferred@781989335 after retry RPC after error@478791767) -> (continuation of Deferred@586453895 after retry RPC after error@1389836981) -> (continuation of Deferred@489332661 after retry RPC after error@1328752439) -> (continuation of Deferred@1230896637 after retry RPC after error@573863153) -> (continuation of Deferred@1806336053 after retry RPC after error@1039534312) -> (continuation of Deferred@459454654 after retry RPC after error@451986655) -> (continuation of Deferred@635720860 after retry RPC after error@676899200)))
21:04:49.685 [WARN - pool-40-thread-1] (HashedWheelTimer.java:716) An exception was thrown by TimerTask.
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343) ~[main/:?]
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at java.lang.Thread.run(Thread.java:748) [?:1.8.0_275]
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86)
at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106)
at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198)
at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158)
at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953)
at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923)
at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464)
at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713)
at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701)
at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788)
at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501)
at java.lang.Thread.run(Thread.java:748)
21:04:49.704 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:49.704 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:49.705 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:49.705 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:49.705 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:49.705 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:49.705 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:49.705 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:49.706 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:49.724 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:49.724 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:49.725 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:49.725 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:49.725 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:49.725 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:49.725 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:49.725 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:49.726 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:49.745 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:49.745 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:49.745 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:49.745 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:49.745 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:49.745 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:49.746 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:49.746 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:49.747 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:49.764 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:49.765 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:49.765 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:49.765 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:49.765 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:49.765 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:49.766 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:49.766 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:49.766 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:49.784 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:49.784 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:49.785 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:49.785 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:49.785 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:49.785 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:49.785 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:49.785 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:49.786 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:49.804 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:49.804 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:49.805 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:49.805 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:49.805 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:49.805 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:49.805 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:49.805 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:49.806 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:49.864 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:49.864 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:49.865 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:49.865 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:49.865 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:49.865 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:49.865 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:49.865 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:49.866 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:49.944 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:49.944 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:49.945 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:49.945 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:49.945 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:49.945 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:49.945 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:49.945 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:49.946 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:49.946 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=242), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [99ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [161ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [161ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [162ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [241ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [241ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [242ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [242ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, deferred=Deferred@1753279967(state=PENDING, result=null, callback=(continuation of Deferred@1656161217 after retry RPC after error@423919814) -> (continuation of Deferred@1377542919 after retry RPC after error@1033523854) -> (continuation of Deferred@1734998007 after retry RPC after error@1511393644) -> (continuation of Deferred@1436195506 after retry RPC after error@981183661) -> (continuation of Deferred@310719066 after retry RPC after error@2016543759) -> (continuation of Deferred@2038193841 after retry RPC after error@512072695) -> (continuation of Deferred@631860639 after retry RPC after error@1305675609), errback=(continuation of Deferred@1656161217 after retry RPC after error@423919814) -> (continuation of Deferred@1377542919 after retry RPC after error@1033523854) -> (continuation of Deferred@1734998007 after retry RPC after error@1511393644) -> (continuation of Deferred@1436195506 after retry RPC after error@981183661) -> (continuation of Deferred@310719066 after retry RPC after error@2016543759) -> (continuation of Deferred@2038193841 after retry RPC after error@512072695) -> (continuation of Deferred@631860639 after retry RPC after error@1305675609)))
21:04:49.947 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=242), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [99ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [161ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [161ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [162ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [241ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [241ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [242ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [242ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, deferred=Deferred@1753279967(state=PENDING, result=null, callback=(continuation of Deferred@1656161217 after retry RPC after error@423919814) -> (continuation of Deferred@1377542919 after retry RPC after error@1033523854) -> (continuation of Deferred@1734998007 after retry RPC after error@1511393644) -> (continuation of Deferred@1436195506 after retry RPC after error@981183661) -> (continuation of Deferred@310719066 after retry RPC after error@2016543759) -> (continuation of Deferred@2038193841 after retry RPC after error@512072695) -> (continuation of Deferred@631860639 after retry RPC after error@1305675609), errback=(continuation of Deferred@1656161217 after retry RPC after error@423919814) -> (continuation of Deferred@1377542919 after retry RPC after error@1033523854) -> (continuation of Deferred@1734998007 after retry RPC after error@1511393644) -> (continuation of Deferred@1436195506 after retry RPC after error@981183661) -> (continuation of Deferred@310719066 after retry RPC after error@2016543759) -> (continuation of Deferred@2038193841 after retry RPC after error@512072695) -> (continuation of Deferred@631860639 after retry RPC after error@1305675609)))
21:04:49.964 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:49.964 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:49.965 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:49.965 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:49.965 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:49.965 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:49.965 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:49.965 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:49.966 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:49.984 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:49.984 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:49.984 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:49.985 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:49.985 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:49.985 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:49.985 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:49.985 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:49.986 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:50.005 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:50.005 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:50.005 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:50.006 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:50.006 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:50.007 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:50.007 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:50.007 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:50.007 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:50.024 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:50.025 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:50.025 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:50.025 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:50.025 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:50.025 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:50.025 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:50.025 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:50.026 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:50.044 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:50.044 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:50.045 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:50.045 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:50.045 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:50.045 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:50.045 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:50.045 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:50.046 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:50.084 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:50.084 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:50.085 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:50.085 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:50.085 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:50.085 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:50.085 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:50.085 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:50.086 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:50.104 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:50.104 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:50.104 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:50.104 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:50.104 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:50.105 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:50.105 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:50.105 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:50.106 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:50.144 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:50.144 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:50.145 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:50.145 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:50.145 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:50.145 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:50.145 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:50.145 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:50.146 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:50.146 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=182), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [19ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [43ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [43ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [139ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [141ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [141ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [142ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [179ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [181ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [181ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [182ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, deferred=Deferred@1747798152(state=PENDING, result=null, callback=(continuation of Deferred@529555214 after retry RPC after error@1771815497) -> (continuation of Deferred@1128529056 after retry RPC after error@1093174600) -> (continuation of Deferred@1549654604 after retry RPC after error@1801699844) -> (continuation of Deferred@924423476 after retry RPC after error@849232986) -> (continuation of Deferred@2324479 after retry RPC after error@1965418013) -> (continuation of Deferred@250506680 after retry RPC after error@1318208683) -> (continuation of Deferred@1427459593 after retry RPC after error@257576888), errback=(continuation of Deferred@529555214 after retry RPC after error@1771815497) -> (continuation of Deferred@1128529056 after retry RPC after error@1093174600) -> (continuation of Deferred@1549654604 after retry RPC after error@1801699844) -> (continuation of Deferred@924423476 after retry RPC after error@849232986) -> (continuation of Deferred@2324479 after retry RPC after error@1965418013) -> (continuation of Deferred@250506680 after retry RPC after error@1318208683) -> (continuation of Deferred@1427459593 after retry RPC after error@257576888)))
21:04:50.147 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=182), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [19ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [43ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [43ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [139ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [141ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [141ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [142ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [179ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [181ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [181ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [182ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, deferred=Deferred@1747798152(state=PENDING, result=null, callback=(continuation of Deferred@529555214 after retry RPC after error@1771815497) -> (continuation of Deferred@1128529056 after retry RPC after error@1093174600) -> (continuation of Deferred@1549654604 after retry RPC after error@1801699844) -> (continuation of Deferred@924423476 after retry RPC after error@849232986) -> (continuation of Deferred@2324479 after retry RPC after error@1965418013) -> (continuation of Deferred@250506680 after retry RPC after error@1318208683) -> (continuation of Deferred@1427459593 after retry RPC after error@257576888), errback=(continuation of Deferred@529555214 after retry RPC after error@1771815497) -> (continuation of Deferred@1128529056 after retry RPC after error@1093174600) -> (continuation of Deferred@1549654604 after retry RPC after error@1801699844) -> (continuation of Deferred@924423476 after retry RPC after error@849232986) -> (continuation of Deferred@2324479 after retry RPC after error@1965418013) -> (continuation of Deferred@250506680 after retry RPC after error@1318208683) -> (continuation of Deferred@1427459593 after retry RPC after error@257576888)))
21:04:50.164 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:50.165 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:50.165 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:50.165 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:50.165 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:50.165 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:50.166 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:50.166 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:50.167 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:50.184 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:50.185 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:50.185 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:50.185 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:50.185 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:50.185 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:50.185 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:50.185 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:50.186 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:50.204 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:50.204 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:50.205 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:50.205 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:50.205 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:50.205 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:50.205 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:50.205 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:50.206 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:50.224 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:50.224 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:50.225 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:50.225 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:50.225 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:50.225 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:50.226 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:50.226 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:50.226 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:50.244 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:50.244 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:50.245 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:50.245 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:50.245 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:50.245 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:50.245 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:50.245 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:50.246 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:50.284 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:50.284 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:50.285 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:50.285 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:50.285 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:50.285 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:50.285 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:50.285 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:50.286 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:50.344 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:50.344 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:50.345 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:50.345 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:50.345 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:50.345 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:50.345 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:50.345 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:50.346 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:50.384 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:50.384 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:50.385 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:50.385 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:50.385 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:50.385 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:50.385 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:50.385 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:50.386 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:50.387 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=222), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [3ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [181ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [181ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [182ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [221ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [221ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [222ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [222ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, deferred=Deferred@1648399770(state=PENDING, result=null, callback=(continuation of Deferred@991053969 after retry RPC after error@659771767) -> (continuation of Deferred@57465182 after retry RPC after error@1709601004) -> (continuation of Deferred@1276766721 after retry RPC after error@1912390172) -> (continuation of Deferred@339694420 after retry RPC after error@1107832879) -> (continuation of Deferred@1094878093 after retry RPC after error@1454986651) -> (continuation of Deferred@947001835 after retry RPC after error@997773186) -> (continuation of Deferred@1861451147 after retry RPC after error@1929050179), errback=(continuation of Deferred@991053969 after retry RPC after error@659771767) -> (continuation of Deferred@57465182 after retry RPC after error@1709601004) -> (continuation of Deferred@1276766721 after retry RPC after error@1912390172) -> (continuation of Deferred@339694420 after retry RPC after error@1107832879) -> (continuation of Deferred@1094878093 after retry RPC after error@1454986651) -> (continuation of Deferred@947001835 after retry RPC after error@997773186) -> (continuation of Deferred@1861451147 after retry RPC after error@1929050179)))
21:04:50.387 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=222), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [3ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [181ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [181ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [182ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [221ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [221ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [222ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [222ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, deferred=Deferred@1648399770(state=PENDING, result=null, callback=(continuation of Deferred@991053969 after retry RPC after error@659771767) -> (continuation of Deferred@57465182 after retry RPC after error@1709601004) -> (continuation of Deferred@1276766721 after retry RPC after error@1912390172) -> (continuation of Deferred@339694420 after retry RPC after error@1107832879) -> (continuation of Deferred@1094878093 after retry RPC after error@1454986651) -> (continuation of Deferred@947001835 after retry RPC after error@997773186) -> (continuation of Deferred@1861451147 after retry RPC after error@1929050179), errback=(continuation of Deferred@991053969 after retry RPC after error@659771767) -> (continuation of Deferred@57465182 after retry RPC after error@1709601004) -> (continuation of Deferred@1276766721 after retry RPC after error@1912390172) -> (continuation of Deferred@339694420 after retry RPC after error@1107832879) -> (continuation of Deferred@1094878093 after retry RPC after error@1454986651) -> (continuation of Deferred@947001835 after retry RPC after error@997773186) -> (continuation of Deferred@1861451147 after retry RPC after error@1929050179)))
21:04:50.404 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:50.404 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:50.404 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:50.405 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:50.405 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:50.405 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:50.405 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:50.405 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:50.406 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:50.424 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:50.425 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:50.425 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:50.425 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:50.425 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:50.425 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:50.425 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:50.425 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:50.426 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:50.444 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:50.444 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:50.445 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:50.445 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:50.445 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:50.445 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:50.445 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:50.445 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:50.446 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:50.464 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:50.464 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:50.465 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:50.465 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:50.465 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:50.465 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:50.465 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:50.465 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:50.466 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:50.484 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:50.484 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:50.484 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:50.484 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:50.484 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:50.484 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:50.484 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:50.485 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:50.485 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:50.524 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:50.525 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:50.525 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:50.525 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:50.525 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:50.525 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:50.525 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:50.525 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:50.526 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:50.584 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:50.585 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:50.585 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:50.585 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:50.585 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:50.585 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:50.585 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:50.586 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:50.586 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:50.587 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=183), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [79ms] refreshing cache from master, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [80ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [80ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [181ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [182ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [182ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [183ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, deferred=Deferred@1809216617(state=PENDING, result=null, callback=(continuation of Deferred@1832099523 after retry RPC after error@1601122762) -> (continuation of Deferred@1699043210 after retry RPC after error@1038615159) -> (continuation of Deferred@1281115791 after retry RPC after error@1780207535) -> (continuation of Deferred@623717473 after retry RPC after error@1487628553) -> (continuation of Deferred@1986868397 after retry RPC after error@1111489400) -> (continuation of Deferred@238913914 after retry RPC after error@232610453), errback=(continuation of Deferred@1832099523 after retry RPC after error@1601122762) -> (continuation of Deferred@1699043210 after retry RPC after error@1038615159) -> (continuation of Deferred@1281115791 after retry RPC after error@1780207535) -> (continuation of Deferred@623717473 after retry RPC after error@1487628553) -> (continuation of Deferred@1986868397 after retry RPC after error@1111489400) -> (continuation of Deferred@238913914 after retry RPC after error@232610453)))
21:04:50.587 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=183), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [79ms] refreshing cache from master, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [80ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [80ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [181ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [182ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [182ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [183ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, deferred=Deferred@1809216617(state=PENDING, result=null, callback=(continuation of Deferred@1832099523 after retry RPC after error@1601122762) -> (continuation of Deferred@1699043210 after retry RPC after error@1038615159) -> (continuation of Deferred@1281115791 after retry RPC after error@1780207535) -> (continuation of Deferred@623717473 after retry RPC after error@1487628553) -> (continuation of Deferred@1986868397 after retry RPC after error@1111489400) -> (continuation of Deferred@238913914 after retry RPC after error@232610453), errback=(continuation of Deferred@1832099523 after retry RPC after error@1601122762) -> (continuation of Deferred@1699043210 after retry RPC after error@1038615159) -> (continuation of Deferred@1281115791 after retry RPC after error@1780207535) -> (continuation of Deferred@623717473 after retry RPC after error@1487628553) -> (continuation of Deferred@1986868397 after retry RPC after error@1111489400) -> (continuation of Deferred@238913914 after retry RPC after error@232610453)))
21:04:50.604 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:50.604 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:50.605 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:50.605 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:50.605 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:50.605 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:50.605 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:50.605 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:50.606 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:50.624 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:50.624 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:50.625 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:50.625 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:50.625 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:50.625 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:50.625 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:50.625 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:50.627 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:50.644 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:50.644 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:50.645 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:50.645 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:50.645 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:50.645 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:50.645 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:50.645 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:50.646 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:50.664 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:50.664 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:50.665 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:50.665 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:50.665 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:50.665 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:50.665 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:50.665 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:50.666 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:50.684 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:50.684 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:50.684 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:50.684 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:50.684 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:50.685 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:50.685 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:50.685 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:50.686 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:50.704 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:50.705 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:50.705 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:50.705 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:50.705 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:50.705 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:50.705 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:50.705 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:50.706 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:50.764 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:50.765 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:50.765 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:50.765 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:50.765 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:50.765 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:50.765 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:50.766 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:50.766 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:50.864 [DEBUG - pool-40-thread-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=259), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [23ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [59ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [79ms] refreshing cache from master, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [161ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [161ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [162ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, deferred=Deferred@836127770(state=PENDING, result=null, callback=(continuation of Deferred@879214311 after retry RPC after error@1932911219) -> (continuation of Deferred@34393869 after retry RPC after error@28964298) -> (continuation of Deferred@136377244 after retry RPC after error@1415233169) -> (continuation of Deferred@818575280 after retry RPC after error@2123613126) -> (continuation of Deferred@1906336667 after retry RPC after error@1880089864) -> (continuation of Deferred@320568142 after retry RPC after error@1937119000) -> (continuation of Deferred@20454998 after retry RPC after error@244354268), errback=(continuation of Deferred@879214311 after retry RPC after error@1932911219) -> (continuation of Deferred@34393869 after retry RPC after error@28964298) -> (continuation of Deferred@136377244 after retry RPC after error@1415233169) -> (continuation of Deferred@818575280 after retry RPC after error@2123613126) -> (continuation of Deferred@1906336667 after retry RPC after error@1880089864) -> (continuation of Deferred@320568142 after retry RPC after error@1937119000) -> (continuation of Deferred@20454998 after retry RPC after error@244354268)))
21:04:50.865 [WARN - pool-40-thread-1] (HashedWheelTimer.java:716) An exception was thrown by TimerTask.
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343) ~[main/:?]
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at java.lang.Thread.run(Thread.java:748) [?:1.8.0_275]
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86)
at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106)
at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198)
at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158)
at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953)
at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923)
at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464)
at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713)
at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701)
at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788)
at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501)
at java.lang.Thread.run(Thread.java:748)
21:04:50.884 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:50.885 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:50.885 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:50.885 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:50.885 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:50.885 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:50.885 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:50.885 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:50.886 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:50.904 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:50.904 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:50.905 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:50.905 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:50.905 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:50.905 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:50.905 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:50.905 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:50.906 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:50.924 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:50.924 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:50.925 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:50.925 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:50.925 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:50.925 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:50.926 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:50.926 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:50.926 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:50.944 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:50.944 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:50.945 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:50.945 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:50.945 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:50.945 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:50.945 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:50.945 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:50.946 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:50.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:50.948935 24071 master_runner.cc:186] Error getting master registration for 127.22.168.62:37239: Network error: Client connection negotiation failed: client connection to 127.22.168.62:37239: connect: Connection refused (error 111),
21:04:50.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:50.949170 24071 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
21:04:50.964 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:50.964 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:50.965 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:50.965 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:50.965 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:50.965 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:50.965 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:50.965 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:50.966 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:50.984 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:50.985 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:50.985 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:50.985 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:50.985 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:50.985 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:50.985 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:50.986 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:50.987 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:51.024 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:51.025 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:51.025 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:51.025 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:51.025 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:51.025 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:51.025 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:51.026 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:51.027 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:51.044 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:51.044 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:51.045 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:51.045 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:51.045 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:51.045 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:51.045 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:51.045 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:51.046 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:51.047 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=162), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [161ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [161ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [162ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, deferred=Deferred@1438972918(state=PENDING, result=null, callback=(continuation of Deferred@639196953 after retry RPC after error@1911105872) -> (continuation of Deferred@659859078 after retry RPC after error@1438507729) -> (continuation of Deferred@959487720 after retry RPC after error@19632951) -> (continuation of Deferred@55145730 after retry RPC after error@962110672) -> (continuation of Deferred@69384909 after retry RPC after error@1498231637) -> (continuation of Deferred@291814427 after retry RPC after error@1843852483), errback=(continuation of Deferred@639196953 after retry RPC after error@1911105872) -> (continuation of Deferred@659859078 after retry RPC after error@1438507729) -> (continuation of Deferred@959487720 after retry RPC after error@19632951) -> (continuation of Deferred@55145730 after retry RPC after error@962110672) -> (continuation of Deferred@69384909 after retry RPC after error@1498231637) -> (continuation of Deferred@291814427 after retry RPC after error@1843852483)))
21:04:51.047 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=162), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [161ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [161ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [162ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, deferred=Deferred@1438972918(state=PENDING, result=null, callback=(continuation of Deferred@639196953 after retry RPC after error@1911105872) -> (continuation of Deferred@659859078 after retry RPC after error@1438507729) -> (continuation of Deferred@959487720 after retry RPC after error@19632951) -> (continuation of Deferred@55145730 after retry RPC after error@962110672) -> (continuation of Deferred@69384909 after retry RPC after error@1498231637) -> (continuation of Deferred@291814427 after retry RPC after error@1843852483), errback=(continuation of Deferred@639196953 after retry RPC after error@1911105872) -> (continuation of Deferred@659859078 after retry RPC after error@1438507729) -> (continuation of Deferred@959487720 after retry RPC after error@19632951) -> (continuation of Deferred@55145730 after retry RPC after error@962110672) -> (continuation of Deferred@69384909 after retry RPC after error@1498231637) -> (continuation of Deferred@291814427 after retry RPC after error@1843852483)))
21:04:51.064 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:51.064 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:51.065 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:51.065 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:51.065 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:51.065 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:51.065 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:51.065 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:51.066 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:51.084 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:51.084 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:51.084 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:51.084 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:51.085 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:51.085 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:51.085 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:51.085 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:51.085 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:51.104 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:51.105 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:51.105 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:51.105 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:51.105 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:51.105 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:51.105 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:51.105 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:51.106 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:51.124 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:51.124 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:51.125 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:51.125 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:51.125 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:51.125 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:51.125 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:51.125 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:51.126 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:51.144 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:51.144 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:51.145 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:51.145 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:51.145 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:51.145 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:51.145 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:51.145 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:51.146 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:51.164 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:51.164 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:51.164 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:51.164 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:51.165 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:51.165 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:51.165 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:51.165 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:51.165 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:51.224 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:51.224 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:51.225 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:51.225 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:51.225 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:51.225 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:51.225 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:51.225 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:51.226 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:51.284 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:51.284 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:51.285 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:51.285 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:51.285 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:51.285 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:51.285 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:51.285 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:51.286 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:51.286 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=222), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [19ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [21ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [99ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [101ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [159ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [161ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [161ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [161ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [221ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [221ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [221ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [222ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, deferred=Deferred@828192367(state=PENDING, result=null, callback=(continuation of Deferred@1379245047 after retry RPC after error@1545459137) -> (continuation of Deferred@1333827877 after retry RPC after error@389423348) -> (continuation of Deferred@1772288966 after retry RPC after error@1529055470) -> (continuation of Deferred@279014174 after retry RPC after error@917952638) -> (continuation of Deferred@1211394469 after retry RPC after error@261614773) -> (continuation of Deferred@343248834 after retry RPC after error@343986202) -> (continuation of Deferred@2140338798 after retry RPC after error@1339052909), errback=(continuation of Deferred@1379245047 after retry RPC after error@1545459137) -> (continuation of Deferred@1333827877 after retry RPC after error@389423348) -> (continuation of Deferred@1772288966 after retry RPC after error@1529055470) -> (continuation of Deferred@279014174 after retry RPC after error@917952638) -> (continuation of Deferred@1211394469 after retry RPC after error@261614773) -> (continuation of Deferred@343248834 after retry RPC after error@343986202) -> (continuation of Deferred@2140338798 after retry RPC after error@1339052909)))
21:04:51.287 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=222), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [19ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [21ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [99ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [101ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [159ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [161ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [161ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [161ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [221ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [221ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [221ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [222ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, deferred=Deferred@828192367(state=PENDING, result=null, callback=(continuation of Deferred@1379245047 after retry RPC after error@1545459137) -> (continuation of Deferred@1333827877 after retry RPC after error@389423348) -> (continuation of Deferred@1772288966 after retry RPC after error@1529055470) -> (continuation of Deferred@279014174 after retry RPC after error@917952638) -> (continuation of Deferred@1211394469 after retry RPC after error@261614773) -> (continuation of Deferred@343248834 after retry RPC after error@343986202) -> (continuation of Deferred@2140338798 after retry RPC after error@1339052909), errback=(continuation of Deferred@1379245047 after retry RPC after error@1545459137) -> (continuation of Deferred@1333827877 after retry RPC after error@389423348) -> (continuation of Deferred@1772288966 after retry RPC after error@1529055470) -> (continuation of Deferred@279014174 after retry RPC after error@917952638) -> (continuation of Deferred@1211394469 after retry RPC after error@261614773) -> (continuation of Deferred@343248834 after retry RPC after error@343986202) -> (continuation of Deferred@2140338798 after retry RPC after error@1339052909)))
21:04:51.304 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:51.304 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:51.304 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:51.304 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:51.305 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:51.305 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:51.305 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:51.305 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:51.306 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:51.324 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:51.325 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:51.325 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:51.325 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:51.325 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:51.325 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:51.325 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:51.325 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:51.326 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:51.344 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:51.344 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:51.345 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:51.345 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:51.345 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:51.345 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:51.345 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:51.345 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:51.346 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:51.364 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:51.364 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:51.365 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:51.365 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:51.365 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:51.365 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:51.365 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:51.365 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:51.367 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:51.384 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:51.384 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:51.384 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:51.384 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:51.384 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:51.385 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:51.385 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:51.385 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:51.386 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:51.404 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:51.404 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:51.405 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:51.405 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:51.405 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:51.405 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:51.405 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:51.405 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:51.406 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:51.464 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:51.465 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:51.465 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:51.465 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:51.465 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:51.465 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:51.465 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:51.465 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:51.466 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:51.467 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=163), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [21ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [23ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [41ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [43ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [61ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [63ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [64ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [101ms] refreshing cache from master, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [103ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [161ms] refreshing cache from master, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [162ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [162ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [162ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [162ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [163ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [164ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, deferred=Deferred@603850720(state=PENDING, result=null, callback=(continuation of Deferred@1870330570 after retry RPC after error@1654176002) -> (continuation of Deferred@531529621 after retry RPC after error@1279048327) -> (continuation of Deferred@1948824928 after retry RPC after error@1407144526) -> (continuation of Deferred@1071766903 after retry RPC after error@379629710) -> (continuation of Deferred@663917711 after retry RPC after error@242679569) -> (continuation of Deferred@1062044109 after retry RPC after error@754610794), errback=(continuation of Deferred@1870330570 after retry RPC after error@1654176002) -> (continuation of Deferred@531529621 after retry RPC after error@1279048327) -> (continuation of Deferred@1948824928 after retry RPC after error@1407144526) -> (continuation of Deferred@1071766903 after retry RPC after error@379629710) -> (continuation of Deferred@663917711 after retry RPC after error@242679569) -> (continuation of Deferred@1062044109 after retry RPC after error@754610794)))
21:04:51.467 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=163), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [21ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [23ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [41ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [43ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [61ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [63ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [64ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [101ms] refreshing cache from master, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [103ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [161ms] refreshing cache from master, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [162ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [162ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [162ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [162ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [163ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [164ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, deferred=Deferred@603850720(state=PENDING, result=null, callback=(continuation of Deferred@1870330570 after retry RPC after error@1654176002) -> (continuation of Deferred@531529621 after retry RPC after error@1279048327) -> (continuation of Deferred@1948824928 after retry RPC after error@1407144526) -> (continuation of Deferred@1071766903 after retry RPC after error@379629710) -> (continuation of Deferred@663917711 after retry RPC after error@242679569) -> (continuation of Deferred@1062044109 after retry RPC after error@754610794), errback=(continuation of Deferred@1870330570 after retry RPC after error@1654176002) -> (continuation of Deferred@531529621 after retry RPC after error@1279048327) -> (continuation of Deferred@1948824928 after retry RPC after error@1407144526) -> (continuation of Deferred@1071766903 after retry RPC after error@379629710) -> (continuation of Deferred@663917711 after retry RPC after error@242679569) -> (continuation of Deferred@1062044109 after retry RPC after error@754610794)))
21:04:51.484 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:51.484 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:51.485 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:51.485 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:51.485 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:51.485 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:51.485 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:51.486 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:51.486 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:51.504 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:51.504 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:51.505 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:51.505 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:51.505 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:51.505 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:51.505 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:51.505 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:51.506 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:51.524 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:51.524 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:51.524 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:51.524 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:51.524 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:51.525 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:51.525 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:51.525 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:51.526 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:51.544 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:51.545 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:51.545 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:51.545 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:51.545 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:51.546 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:51.546 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:51.546 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:51.546 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:51.564 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:51.565 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:51.565 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:51.565 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:51.565 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:51.565 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:51.565 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:51.565 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:51.567 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:51.584 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:51.585 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:51.585 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:51.585 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:51.585 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:51.585 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:51.585 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:51.585 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:51.586 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:51.644 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:51.645 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:51.645 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:51.645 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:51.645 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:51.645 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:51.646 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:51.646 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:51.646 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:51.647 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=162), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [39ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [40ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [161ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [162ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [162ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, deferred=Deferred@728136861(state=PENDING, result=null, callback=(continuation of Deferred@1424997311 after retry RPC after error@1811601828) -> (continuation of Deferred@1751979243 after retry RPC after error@1895239663) -> (continuation of Deferred@1500278797 after retry RPC after error@1692837245) -> (continuation of Deferred@1680617286 after retry RPC after error@745331803) -> (continuation of Deferred@372915651 after retry RPC after error@1202110260) -> (continuation of Deferred@1123417753 after retry RPC after error@145849496), errback=(continuation of Deferred@1424997311 after retry RPC after error@1811601828) -> (continuation of Deferred@1751979243 after retry RPC after error@1895239663) -> (continuation of Deferred@1500278797 after retry RPC after error@1692837245) -> (continuation of Deferred@1680617286 after retry RPC after error@745331803) -> (continuation of Deferred@372915651 after retry RPC after error@1202110260) -> (continuation of Deferred@1123417753 after retry RPC after error@145849496)))
21:04:51.647 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=162), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [39ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [40ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [161ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [162ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [162ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, deferred=Deferred@728136861(state=PENDING, result=null, callback=(continuation of Deferred@1424997311 after retry RPC after error@1811601828) -> (continuation of Deferred@1751979243 after retry RPC after error@1895239663) -> (continuation of Deferred@1500278797 after retry RPC after error@1692837245) -> (continuation of Deferred@1680617286 after retry RPC after error@745331803) -> (continuation of Deferred@372915651 after retry RPC after error@1202110260) -> (continuation of Deferred@1123417753 after retry RPC after error@145849496), errback=(continuation of Deferred@1424997311 after retry RPC after error@1811601828) -> (continuation of Deferred@1751979243 after retry RPC after error@1895239663) -> (continuation of Deferred@1500278797 after retry RPC after error@1692837245) -> (continuation of Deferred@1680617286 after retry RPC after error@745331803) -> (continuation of Deferred@372915651 after retry RPC after error@1202110260) -> (continuation of Deferred@1123417753 after retry RPC after error@145849496)))
21:04:51.664 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:51.665 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:51.665 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:51.665 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:51.665 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:51.665 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:51.665 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:51.665 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:51.666 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:51.684 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:51.684 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:51.685 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:51.685 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:51.685 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:51.685 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:51.685 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:51.685 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:51.686 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:51.704 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:51.704 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:51.705 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:51.705 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:51.705 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:51.705 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:51.705 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:51.705 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:51.706 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:51.724 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:51.725 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:51.725 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:51.725 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:51.725 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:51.725 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:51.726 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:51.726 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:51.726 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:51.744 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:51.744 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:51.745 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:51.745 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:51.745 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:51.745 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:51.745 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:51.745 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:51.746 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:51.764 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:51.764 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:51.765 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:51.765 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:51.765 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:51.765 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:51.765 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:51.765 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:51.766 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:51.804 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:51.804 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:51.805 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:51.805 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:51.805 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:51.805 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:51.805 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:51.805 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:51.806 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:51.864 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:51.864 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:51.865 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:51.865 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:51.865 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:51.865 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:51.865 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:51.865 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:51.866 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:51.866 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=201), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [39ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [141ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [141ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [142ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [201ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [201ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [202ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, deferred=Deferred@569727487(state=PENDING, result=null, callback=(continuation of Deferred@684255895 after retry RPC after error@133395389) -> (continuation of Deferred@904443758 after retry RPC after error@1046030989) -> (continuation of Deferred@1370157955 after retry RPC after error@1297620112) -> (continuation of Deferred@1031504274 after retry RPC after error@1241631218) -> (continuation of Deferred@1286675930 after retry RPC after error@1098916928) -> (continuation of Deferred@538331891 after retry RPC after error@1679575386) -> (continuation of Deferred@537380814 after retry RPC after error@1625642410), errback=(continuation of Deferred@684255895 after retry RPC after error@133395389) -> (continuation of Deferred@904443758 after retry RPC after error@1046030989) -> (continuation of Deferred@1370157955 after retry RPC after error@1297620112) -> (continuation of Deferred@1031504274 after retry RPC after error@1241631218) -> (continuation of Deferred@1286675930 after retry RPC after error@1098916928) -> (continuation of Deferred@538331891 after retry RPC after error@1679575386) -> (continuation of Deferred@537380814 after retry RPC after error@1625642410)))
21:04:51.867 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=201), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [39ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [141ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [141ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [142ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [201ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [201ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [202ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, deferred=Deferred@569727487(state=PENDING, result=null, callback=(continuation of Deferred@684255895 after retry RPC after error@133395389) -> (continuation of Deferred@904443758 after retry RPC after error@1046030989) -> (continuation of Deferred@1370157955 after retry RPC after error@1297620112) -> (continuation of Deferred@1031504274 after retry RPC after error@1241631218) -> (continuation of Deferred@1286675930 after retry RPC after error@1098916928) -> (continuation of Deferred@538331891 after retry RPC after error@1679575386) -> (continuation of Deferred@537380814 after retry RPC after error@1625642410), errback=(continuation of Deferred@684255895 after retry RPC after error@133395389) -> (continuation of Deferred@904443758 after retry RPC after error@1046030989) -> (continuation of Deferred@1370157955 after retry RPC after error@1297620112) -> (continuation of Deferred@1031504274 after retry RPC after error@1241631218) -> (continuation of Deferred@1286675930 after retry RPC after error@1098916928) -> (continuation of Deferred@538331891 after retry RPC after error@1679575386) -> (continuation of Deferred@537380814 after retry RPC after error@1625642410)))
21:04:51.884 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:51.884 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:51.885 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:51.885 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:51.885 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:51.885 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:51.885 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:51.885 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:51.886 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:51.904 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:51.905 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:51.905 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:51.905 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:51.905 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:51.905 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:51.905 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:51.905 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:51.906 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:51.924 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:51.924 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:51.925 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:51.925 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:51.925 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:51.925 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:51.925 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:51.925 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:51.926 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:51.944 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:51.944 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:51.945 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:51.945 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:51.945 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:51.945 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:51.945 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:51.945 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:51.946 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:51.964 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:51.964 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:51.965 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:51.965 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:51.965 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:51.965 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:51.965 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:51.965 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:51.966 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:52.004 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:52.004 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:52.004 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:52.004 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:52.004 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:52.005 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:52.005 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:52.005 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:52.006 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:52.024 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:52.025 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:52.025 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:52.025 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:52.025 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:52.025 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:52.025 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:52.025 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:52.026 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:52.124 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:52.124 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:52.124 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:52.124 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:52.125 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:52.125 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:52.125 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:52.125 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:52.126 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:52.126 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=242), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [119ms] refreshing cache from master, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [120ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [141ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [141ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [142ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [239ms] refreshing cache from master, [239ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [241ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [241ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [241ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [242ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, deferred=Deferred@1581952105(state=PENDING, result=null, callback=(continuation of Deferred@1811975278 after retry RPC after error@7144003) -> (continuation of Deferred@1443467371 after retry RPC after error@1571505959) -> (continuation of Deferred@899732224 after retry RPC after error@1563930861) -> (continuation of Deferred@1193798120 after retry RPC after error@1085410388) -> (continuation of Deferred@1941565329 after retry RPC after error@325683667) -> (continuation of Deferred@1803111167 after retry RPC after error@997800808) -> (continuation of Deferred@117477105 after retry RPC after error@1484306643), errback=(continuation of Deferred@1811975278 after retry RPC after error@7144003) -> (continuation of Deferred@1443467371 after retry RPC after error@1571505959) -> (continuation of Deferred@899732224 after retry RPC after error@1563930861) -> (continuation of Deferred@1193798120 after retry RPC after error@1085410388) -> (continuation of Deferred@1941565329 after retry RPC after error@325683667) -> (continuation of Deferred@1803111167 after retry RPC after error@997800808) -> (continuation of Deferred@117477105 after retry RPC after error@1484306643)))
21:04:52.127 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=242), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [119ms] refreshing cache from master, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [120ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [141ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [141ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [142ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [239ms] refreshing cache from master, [239ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [241ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [241ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [241ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [242ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, deferred=Deferred@1581952105(state=PENDING, result=null, callback=(continuation of Deferred@1811975278 after retry RPC after error@7144003) -> (continuation of Deferred@1443467371 after retry RPC after error@1571505959) -> (continuation of Deferred@899732224 after retry RPC after error@1563930861) -> (continuation of Deferred@1193798120 after retry RPC after error@1085410388) -> (continuation of Deferred@1941565329 after retry RPC after error@325683667) -> (continuation of Deferred@1803111167 after retry RPC after error@997800808) -> (continuation of Deferred@117477105 after retry RPC after error@1484306643), errback=(continuation of Deferred@1811975278 after retry RPC after error@7144003) -> (continuation of Deferred@1443467371 after retry RPC after error@1571505959) -> (continuation of Deferred@899732224 after retry RPC after error@1563930861) -> (continuation of Deferred@1193798120 after retry RPC after error@1085410388) -> (continuation of Deferred@1941565329 after retry RPC after error@325683667) -> (continuation of Deferred@1803111167 after retry RPC after error@997800808) -> (continuation of Deferred@117477105 after retry RPC after error@1484306643)))
21:04:52.144 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:52.145 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:52.145 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:52.145 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:52.145 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:52.145 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:52.145 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:52.145 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:52.146 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:52.164 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:52.164 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:52.165 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:52.165 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:52.165 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:52.165 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:52.165 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:52.165 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:52.166 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:52.184 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:52.184 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:52.185 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:52.185 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:52.185 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:52.185 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:52.185 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:52.185 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:52.186 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:52.204 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:52.204 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:52.204 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:52.204 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:52.205 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:52.205 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:52.205 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:52.205 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:52.206 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:52.224 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:52.225 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:52.225 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:52.225 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:52.225 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:52.225 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:52.225 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:52.225 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:52.227 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:52.264 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:52.265 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:52.265 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:52.265 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:52.265 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:52.265 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:52.265 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:52.266 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:52.266 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:52.324 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:52.325 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:52.325 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:52.325 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:52.325 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:52.325 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:52.326 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:52.326 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:52.326 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:52.327 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=182), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [59ms] refreshing cache from master, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [181ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [181ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [182ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [183ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, deferred=Deferred@849950987(state=PENDING, result=null, callback=(continuation of Deferred@737208085 after retry RPC after error@433231089) -> (continuation of Deferred@2145734395 after retry RPC after error@75448685) -> (continuation of Deferred@681301688 after retry RPC after error@627288646) -> (continuation of Deferred@1889839751 after retry RPC after error@246348247) -> (continuation of Deferred@2029437100 after retry RPC after error@1090479214) -> (continuation of Deferred@358223460 after retry RPC after error@1816341849), errback=(continuation of Deferred@737208085 after retry RPC after error@433231089) -> (continuation of Deferred@2145734395 after retry RPC after error@75448685) -> (continuation of Deferred@681301688 after retry RPC after error@627288646) -> (continuation of Deferred@1889839751 after retry RPC after error@246348247) -> (continuation of Deferred@2029437100 after retry RPC after error@1090479214) -> (continuation of Deferred@358223460 after retry RPC after error@1816341849)))
21:04:52.327 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=182), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [59ms] refreshing cache from master, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [181ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [181ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [182ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [183ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, deferred=Deferred@849950987(state=PENDING, result=null, callback=(continuation of Deferred@737208085 after retry RPC after error@433231089) -> (continuation of Deferred@2145734395 after retry RPC after error@75448685) -> (continuation of Deferred@681301688 after retry RPC after error@627288646) -> (continuation of Deferred@1889839751 after retry RPC after error@246348247) -> (continuation of Deferred@2029437100 after retry RPC after error@1090479214) -> (continuation of Deferred@358223460 after retry RPC after error@1816341849), errback=(continuation of Deferred@737208085 after retry RPC after error@433231089) -> (continuation of Deferred@2145734395 after retry RPC after error@75448685) -> (continuation of Deferred@681301688 after retry RPC after error@627288646) -> (continuation of Deferred@1889839751 after retry RPC after error@246348247) -> (continuation of Deferred@2029437100 after retry RPC after error@1090479214) -> (continuation of Deferred@358223460 after retry RPC after error@1816341849)))
21:04:52.344 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:52.345 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:52.345 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:52.345 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:52.345 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:52.345 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:52.345 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:52.345 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:52.346 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:52.364 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:52.364 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:52.365 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:52.365 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:52.365 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:52.365 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:52.365 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:52.365 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:52.366 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:52.384 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:52.384 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:52.385 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:52.385 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:52.386 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:52.386 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:52.386 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:52.386 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:52.386 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:52.404 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:52.404 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:52.404 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:52.404 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:52.405 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:52.405 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:52.405 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:52.405 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:52.406 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:52.424 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:52.425 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:52.425 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:52.425 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:52.425 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:52.425 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:52.425 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:52.426 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:52.426 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:52.444 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:52.444 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:52.445 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:52.445 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:52.445 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:52.445 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:52.445 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:52.445 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:52.446 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:52.504 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:52.504 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:52.505 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:52.505 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:52.505 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:52.505 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:52.505 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:52.505 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:52.506 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:52.564 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:52.565 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:52.565 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:52.565 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:52.565 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:52.565 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:52.566 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:52.566 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:52.566 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:52.566 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=222), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [59ms] refreshing cache from master, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [161ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [161ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [162ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [222ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [222ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [222ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [222ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, deferred=Deferred@172812320(state=PENDING, result=null, callback=(continuation of Deferred@909465098 after retry RPC after error@199179837) -> (continuation of Deferred@749581646 after retry RPC after error@1095582995) -> (continuation of Deferred@1574227510 after retry RPC after error@789069159) -> (continuation of Deferred@1859054093 after retry RPC after error@1166040858) -> (continuation of Deferred@1054772450 after retry RPC after error@1356037467) -> (continuation of Deferred@1230269390 after retry RPC after error@241931633) -> (continuation of Deferred@1150857666 after retry RPC after error@258614703), errback=(continuation of Deferred@909465098 after retry RPC after error@199179837) -> (continuation of Deferred@749581646 after retry RPC after error@1095582995) -> (continuation of Deferred@1574227510 after retry RPC after error@789069159) -> (continuation of Deferred@1859054093 after retry RPC after error@1166040858) -> (continuation of Deferred@1054772450 after retry RPC after error@1356037467) -> (continuation of Deferred@1230269390 after retry RPC after error@241931633) -> (continuation of Deferred@1150857666 after retry RPC after error@258614703)))
21:04:52.567 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=222), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [59ms] refreshing cache from master, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [161ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [161ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [162ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [222ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [222ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [222ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [222ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, deferred=Deferred@172812320(state=PENDING, result=null, callback=(continuation of Deferred@909465098 after retry RPC after error@199179837) -> (continuation of Deferred@749581646 after retry RPC after error@1095582995) -> (continuation of Deferred@1574227510 after retry RPC after error@789069159) -> (continuation of Deferred@1859054093 after retry RPC after error@1166040858) -> (continuation of Deferred@1054772450 after retry RPC after error@1356037467) -> (continuation of Deferred@1230269390 after retry RPC after error@241931633) -> (continuation of Deferred@1150857666 after retry RPC after error@258614703), errback=(continuation of Deferred@909465098 after retry RPC after error@199179837) -> (continuation of Deferred@749581646 after retry RPC after error@1095582995) -> (continuation of Deferred@1574227510 after retry RPC after error@789069159) -> (continuation of Deferred@1859054093 after retry RPC after error@1166040858) -> (continuation of Deferred@1054772450 after retry RPC after error@1356037467) -> (continuation of Deferred@1230269390 after retry RPC after error@241931633) -> (continuation of Deferred@1150857666 after retry RPC after error@258614703)))
21:04:52.584 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:52.584 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:52.585 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:52.585 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:52.585 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:52.585 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:52.585 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:52.585 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:52.586 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:52.604 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:52.604 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:52.605 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:52.605 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:52.605 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:52.605 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:52.605 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:52.605 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:52.606 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:52.624 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:52.624 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:52.624 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:52.624 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:52.625 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:52.625 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:52.625 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:52.625 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:52.625 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:52.644 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:52.644 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:52.645 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:52.645 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:52.645 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:52.645 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:52.645 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:52.645 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:52.646 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:52.664 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:52.664 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:52.665 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:52.665 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:52.665 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:52.665 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:52.665 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:52.665 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:52.666 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:52.684 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:52.684 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:52.684 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:52.684 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:52.685 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:52.685 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:52.685 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:52.685 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:52.685 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:52.724 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:52.725 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:52.725 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:52.725 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:52.726 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:52.726 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:52.726 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:52.726 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:52.726 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:52.824 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:52.824 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:52.825 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:52.825 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:52.825 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:52.825 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:52.825 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:52.825 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:52.826 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:52.826 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=241), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [39ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [41ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [99ms] refreshing cache from master, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [101ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [142ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [142ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [142ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [241ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [241ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [242ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [242ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, deferred=Deferred@710880126(state=PENDING, result=null, callback=(continuation of Deferred@321659881 after retry RPC after error@19782975) -> (continuation of Deferred@786775666 after retry RPC after error@2108879883) -> (continuation of Deferred@267629375 after retry RPC after error@1616397275) -> (continuation of Deferred@1741285132 after retry RPC after error@229725522) -> (continuation of Deferred@533296528 after retry RPC after error@1034791721) -> (continuation of Deferred@359979543 after retry RPC after error@317195822) -> (continuation of Deferred@1188660246 after retry RPC after error@377326343), errback=(continuation of Deferred@321659881 after retry RPC after error@19782975) -> (continuation of Deferred@786775666 after retry RPC after error@2108879883) -> (continuation of Deferred@267629375 after retry RPC after error@1616397275) -> (continuation of Deferred@1741285132 after retry RPC after error@229725522) -> (continuation of Deferred@533296528 after retry RPC after error@1034791721) -> (continuation of Deferred@359979543 after retry RPC after error@317195822) -> (continuation of Deferred@1188660246 after retry RPC after error@377326343)))
21:04:52.827 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=241), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [39ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [41ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [99ms] refreshing cache from master, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [101ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [142ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [142ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [142ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [241ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [241ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [242ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [242ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, deferred=Deferred@710880126(state=PENDING, result=null, callback=(continuation of Deferred@321659881 after retry RPC after error@19782975) -> (continuation of Deferred@786775666 after retry RPC after error@2108879883) -> (continuation of Deferred@267629375 after retry RPC after error@1616397275) -> (continuation of Deferred@1741285132 after retry RPC after error@229725522) -> (continuation of Deferred@533296528 after retry RPC after error@1034791721) -> (continuation of Deferred@359979543 after retry RPC after error@317195822) -> (continuation of Deferred@1188660246 after retry RPC after error@377326343), errback=(continuation of Deferred@321659881 after retry RPC after error@19782975) -> (continuation of Deferred@786775666 after retry RPC after error@2108879883) -> (continuation of Deferred@267629375 after retry RPC after error@1616397275) -> (continuation of Deferred@1741285132 after retry RPC after error@229725522) -> (continuation of Deferred@533296528 after retry RPC after error@1034791721) -> (continuation of Deferred@359979543 after retry RPC after error@317195822) -> (continuation of Deferred@1188660246 after retry RPC after error@377326343)))
21:04:52.845 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:52.845 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:52.845 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:52.845 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:52.846 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:52.846 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:52.846 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:52.846 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:52.846 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:52.864 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:52.865 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:52.865 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:52.865 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:52.865 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:52.865 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:52.865 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:52.865 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:52.866 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:52.884 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:52.884 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:52.885 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:52.885 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:52.885 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:52.885 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:52.885 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:52.885 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:52.886 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:52.904 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:52.904 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:52.905 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:52.905 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:52.905 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:52.905 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:52.905 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:52.905 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:52.906 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:52.925 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:52.925 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:52.925 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:52.925 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:52.926 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:52.926 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:52.926 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:52.926 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:52.927 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:52.944 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:52.945 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:52.945 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:52.945 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:52.945 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:52.945 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:52.946 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:52.946 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:52.947 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:52.964 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:52.964 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:52.965 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:52.965 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:52.965 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:52.965 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:52.966 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:52.966 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:52.966 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:52.984 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:52.984 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:52.985 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:52.985 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:52.985 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:52.985 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:52.985 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:52.985 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:52.986 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:52.987 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=141), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [83ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [141ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [141ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [142ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, deferred=Deferred@878914003(state=PENDING, result=null, callback=(continuation of Deferred@1545387219 after retry RPC after error@1378875018) -> (continuation of Deferred@1123263187 after retry RPC after error@1869821679) -> (continuation of Deferred@1446049270 after retry RPC after error@2072429084) -> (continuation of Deferred@616305285 after retry RPC after error@1376685235) -> (continuation of Deferred@54545473 after retry RPC after error@1318844439) -> (continuation of Deferred@184017787 after retry RPC after error@742038914) -> (continuation of Deferred@756674048 after retry RPC after error@1020518644), errback=(continuation of Deferred@1545387219 after retry RPC after error@1378875018) -> (continuation of Deferred@1123263187 after retry RPC after error@1869821679) -> (continuation of Deferred@1446049270 after retry RPC after error@2072429084) -> (continuation of Deferred@616305285 after retry RPC after error@1376685235) -> (continuation of Deferred@54545473 after retry RPC after error@1318844439) -> (continuation of Deferred@184017787 after retry RPC after error@742038914) -> (continuation of Deferred@756674048 after retry RPC after error@1020518644)))
21:04:52.987 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=141), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [83ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [141ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [141ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [142ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, deferred=Deferred@878914003(state=PENDING, result=null, callback=(continuation of Deferred@1545387219 after retry RPC after error@1378875018) -> (continuation of Deferred@1123263187 after retry RPC after error@1869821679) -> (continuation of Deferred@1446049270 after retry RPC after error@2072429084) -> (continuation of Deferred@616305285 after retry RPC after error@1376685235) -> (continuation of Deferred@54545473 after retry RPC after error@1318844439) -> (continuation of Deferred@184017787 after retry RPC after error@742038914) -> (continuation of Deferred@756674048 after retry RPC after error@1020518644), errback=(continuation of Deferred@1545387219 after retry RPC after error@1378875018) -> (continuation of Deferred@1123263187 after retry RPC after error@1869821679) -> (continuation of Deferred@1446049270 after retry RPC after error@2072429084) -> (continuation of Deferred@616305285 after retry RPC after error@1376685235) -> (continuation of Deferred@54545473 after retry RPC after error@1318844439) -> (continuation of Deferred@184017787 after retry RPC after error@742038914) -> (continuation of Deferred@756674048 after retry RPC after error@1020518644)))
21:04:53.004 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:53.004 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:53.005 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:53.005 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:53.005 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:53.005 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:53.005 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:53.005 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:53.006 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:53.025 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:53.025 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:53.025 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:53.025 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:53.025 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:53.026 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:53.025 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:53.026 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:53.026 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:53.044 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:53.044 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:53.045 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:53.045 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:53.045 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:53.045 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:53.045 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:53.046 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:53.046 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:53.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:53.056529 23569 master_proxy_rpc.cc:177] Failed to determine new leader Master: Timed out: ConnectToClusterRpc(addrs: 127.22.168.62:37239,127.22.168.61:32947,127.22.168.60:33247, num_attempts: 97) passed its deadline: Not found: no leader found: ConnectToClusterRpc(addrs: 127.22.168.62:37239,127.22.168.61:32947,127.22.168.60:33247, num_attempts: 1)
21:04:53.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:53.060029 23569 meta_cache.cc:788] Timed out: LookupRpcById { tablet: '01aa0e192c434b468ec7ed90d48563b3', attempt: 2 } failed: LookupRpcById timed out after deadline expired: LookupRpcById { tablet: '01aa0e192c434b468ec7ed90d48563b3', attempt: 2 } passed its deadline: Timed out: ConnectToClusterRpc(addrs: 127.22.168.62:37239,127.22.168.61:32947,127.22.168.60:33247, num_attempts: 97) passed its deadline: Not found: no leader found: ConnectToClusterRpc(addrs: 127.22.168.62:37239,127.22.168.61:32947,127.22.168.60:33247, num_attempts: 1)
21:04:53.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:53.062155 23569 master_proxy_rpc.cc:203] Re-attempting LookupRpcById request to leader Master (127.22.168.60:33247)
21:04:53.064 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:53.064 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:53.065 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:53.065 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:53.065 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:53.065 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:53.065 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:53.065 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:53.066 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:53.084 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:53.084 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:53.084 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:53.084 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:53.085 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:53.085 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:53.085 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:53.085 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:53.086 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:53.104 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:53.105 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:53.105 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:53.105 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:53.105 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:53.105 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:53.105 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:53.105 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:53.106 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:53.144 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:53.145 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:53.145 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:53.145 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:53.145 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:53.145 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:53.145 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:53.145 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:53.146 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:53.244 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:53.245 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:53.245 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:53.245 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:53.245 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:53.245 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:53.246 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:53.246 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:53.246 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:53.247 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=243), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [79ms] refreshing cache from master, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [141ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [141ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [142ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [241ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [241ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [242ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [242ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, deferred=Deferred@1416269461(state=PENDING, result=null, callback=(continuation of Deferred@1152517682 after retry RPC after error@1715833713) -> (continuation of Deferred@1679431795 after retry RPC after error@1686383683) -> (continuation of Deferred@819144055 after retry RPC after error@2047295007) -> (continuation of Deferred@1315600556 after retry RPC after error@189375254) -> (continuation of Deferred@572053397 after retry RPC after error@1630331407) -> (continuation of Deferred@2090748318 after retry RPC after error@973673480) -> (continuation of Deferred@1581893795 after retry RPC after error@1492350463), errback=(continuation of Deferred@1152517682 after retry RPC after error@1715833713) -> (continuation of Deferred@1679431795 after retry RPC after error@1686383683) -> (continuation of Deferred@819144055 after retry RPC after error@2047295007) -> (continuation of Deferred@1315600556 after retry RPC after error@189375254) -> (continuation of Deferred@572053397 after retry RPC after error@1630331407) -> (continuation of Deferred@2090748318 after retry RPC after error@973673480) -> (continuation of Deferred@1581893795 after retry RPC after error@1492350463)))
21:04:53.247 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=243), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [79ms] refreshing cache from master, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [141ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [141ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [142ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [241ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [241ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [242ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [242ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, deferred=Deferred@1416269461(state=PENDING, result=null, callback=(continuation of Deferred@1152517682 after retry RPC after error@1715833713) -> (continuation of Deferred@1679431795 after retry RPC after error@1686383683) -> (continuation of Deferred@819144055 after retry RPC after error@2047295007) -> (continuation of Deferred@1315600556 after retry RPC after error@189375254) -> (continuation of Deferred@572053397 after retry RPC after error@1630331407) -> (continuation of Deferred@2090748318 after retry RPC after error@973673480) -> (continuation of Deferred@1581893795 after retry RPC after error@1492350463), errback=(continuation of Deferred@1152517682 after retry RPC after error@1715833713) -> (continuation of Deferred@1679431795 after retry RPC after error@1686383683) -> (continuation of Deferred@819144055 after retry RPC after error@2047295007) -> (continuation of Deferred@1315600556 after retry RPC after error@189375254) -> (continuation of Deferred@572053397 after retry RPC after error@1630331407) -> (continuation of Deferred@2090748318 after retry RPC after error@973673480) -> (continuation of Deferred@1581893795 after retry RPC after error@1492350463)))
21:04:53.264 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:53.264 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:53.265 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:53.265 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:53.265 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:53.265 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:53.265 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:53.265 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:53.266 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:53.284 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:53.284 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:53.285 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:53.285 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:53.285 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:53.285 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:53.285 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:53.285 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:53.286 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:53.304 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:53.304 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:53.304 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:53.304 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:53.304 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:53.305 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:53.305 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:53.305 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:53.306 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:53.324 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:53.325 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:53.325 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:53.325 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:53.325 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:53.325 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:53.325 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:53.325 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:53.326 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:53.344 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:53.344 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:53.345 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:53.345 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:53.345 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:53.345 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:53.345 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:53.345 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:53.346 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:53.364 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:53.364 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:53.365 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:53.365 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:53.365 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:53.365 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:53.365 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:53.365 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:53.366 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:53.404 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:53.404 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:53.404 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:53.404 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:53.405 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:53.405 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:53.405 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:53.405 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:53.406 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:53.406 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=141), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [39ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [40ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [139ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [140ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [141ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [141ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, deferred=Deferred@1156459816(state=PENDING, result=null, callback=(continuation of Deferred@1830424503 after retry RPC after error@1059275404) -> (continuation of Deferred@1096888125 after retry RPC after error@1951649108) -> (continuation of Deferred@1233716190 after retry RPC after error@1813716801) -> (continuation of Deferred@608951608 after retry RPC after error@1310236040) -> (continuation of Deferred@1334763622 after retry RPC after error@2069565377) -> (continuation of Deferred@17018767 after retry RPC after error@201164538), errback=(continuation of Deferred@1830424503 after retry RPC after error@1059275404) -> (continuation of Deferred@1096888125 after retry RPC after error@1951649108) -> (continuation of Deferred@1233716190 after retry RPC after error@1813716801) -> (continuation of Deferred@608951608 after retry RPC after error@1310236040) -> (continuation of Deferred@1334763622 after retry RPC after error@2069565377) -> (continuation of Deferred@17018767 after retry RPC after error@201164538)))
21:04:53.406 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=141), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [39ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [40ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [139ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [140ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [141ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [141ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, deferred=Deferred@1156459816(state=PENDING, result=null, callback=(continuation of Deferred@1830424503 after retry RPC after error@1059275404) -> (continuation of Deferred@1096888125 after retry RPC after error@1951649108) -> (continuation of Deferred@1233716190 after retry RPC after error@1813716801) -> (continuation of Deferred@608951608 after retry RPC after error@1310236040) -> (continuation of Deferred@1334763622 after retry RPC after error@2069565377) -> (continuation of Deferred@17018767 after retry RPC after error@201164538), errback=(continuation of Deferred@1830424503 after retry RPC after error@1059275404) -> (continuation of Deferred@1096888125 after retry RPC after error@1951649108) -> (continuation of Deferred@1233716190 after retry RPC after error@1813716801) -> (continuation of Deferred@608951608 after retry RPC after error@1310236040) -> (continuation of Deferred@1334763622 after retry RPC after error@2069565377) -> (continuation of Deferred@17018767 after retry RPC after error@201164538)))
21:04:53.424 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:53.425 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:53.425 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:53.425 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:53.425 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:53.425 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:53.425 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:53.425 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:53.426 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:53.444 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:53.444 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:53.445 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:53.445 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:53.445 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:53.445 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:53.445 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:53.445 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:53.446 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:53.464 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:53.464 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:53.465 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:53.465 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:53.465 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:53.465 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:53.465 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:53.465 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:53.466 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:53.484 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:53.485 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:53.485 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:53.485 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:53.485 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:53.485 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:53.486 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:53.486 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:53.486 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:53.504 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:53.505 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:53.505 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:53.505 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:53.505 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:53.505 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:53.506 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:53.506 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:53.506 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:53.524 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:53.524 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:53.525 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:53.525 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:53.525 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:53.525 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:53.525 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:53.525 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:53.526 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:53.564 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:53.564 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:53.565 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:53.565 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:53.565 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:53.565 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:53.565 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:53.565 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:53.566 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:53.624 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:53.624 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:53.625 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:53.625 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:53.625 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:53.625 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:53.625 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:53.625 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:53.626 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:53.627 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=202), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [141ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [141ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [142ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [201ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [201ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [202ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, deferred=Deferred@548908333(state=PENDING, result=null, callback=(continuation of Deferred@307870052 after retry RPC after error@1692861105) -> (continuation of Deferred@1613955843 after retry RPC after error@2104493293) -> (continuation of Deferred@582811838 after retry RPC after error@2123594962) -> (continuation of Deferred@113267480 after retry RPC after error@2070575169) -> (continuation of Deferred@823672674 after retry RPC after error@1409855615) -> (continuation of Deferred@1356095861 after retry RPC after error@2018816862) -> (continuation of Deferred@147288089 after retry RPC after error@433959478), errback=(continuation of Deferred@307870052 after retry RPC after error@1692861105) -> (continuation of Deferred@1613955843 after retry RPC after error@2104493293) -> (continuation of Deferred@582811838 after retry RPC after error@2123594962) -> (continuation of Deferred@113267480 after retry RPC after error@2070575169) -> (continuation of Deferred@823672674 after retry RPC after error@1409855615) -> (continuation of Deferred@1356095861 after retry RPC after error@2018816862) -> (continuation of Deferred@147288089 after retry RPC after error@433959478)))
21:04:53.627 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=202), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [141ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [141ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [142ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [201ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [201ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [202ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, deferred=Deferred@548908333(state=PENDING, result=null, callback=(continuation of Deferred@307870052 after retry RPC after error@1692861105) -> (continuation of Deferred@1613955843 after retry RPC after error@2104493293) -> (continuation of Deferred@582811838 after retry RPC after error@2123594962) -> (continuation of Deferred@113267480 after retry RPC after error@2070575169) -> (continuation of Deferred@823672674 after retry RPC after error@1409855615) -> (continuation of Deferred@1356095861 after retry RPC after error@2018816862) -> (continuation of Deferred@147288089 after retry RPC after error@433959478), errback=(continuation of Deferred@307870052 after retry RPC after error@1692861105) -> (continuation of Deferred@1613955843 after retry RPC after error@2104493293) -> (continuation of Deferred@582811838 after retry RPC after error@2123594962) -> (continuation of Deferred@113267480 after retry RPC after error@2070575169) -> (continuation of Deferred@823672674 after retry RPC after error@1409855615) -> (continuation of Deferred@1356095861 after retry RPC after error@2018816862) -> (continuation of Deferred@147288089 after retry RPC after error@433959478)))
21:04:53.644 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:53.644 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:53.645 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:53.645 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:53.645 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:53.645 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:53.646 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:53.646 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:53.646 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:53.664 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:53.664 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:53.664 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:53.665 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:53.665 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:53.665 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:53.665 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:53.665 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:53.666 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:53.684 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:53.685 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:53.685 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:53.685 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:53.685 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:53.686 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:53.686 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:53.686 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:53.687 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:53.704 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:53.704 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:53.705 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:53.705 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:53.705 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:53.705 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:53.705 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:53.705 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:53.706 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:53.724 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:53.724 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:53.725 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:53.725 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:53.725 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:53.725 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:53.725 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:53.725 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:53.726 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:53.764 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:53.764 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:53.765 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:53.765 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:53.765 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:53.765 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:53.765 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:53.765 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:53.766 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:53.804 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:53.804 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:53.804 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:53.804 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:53.805 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:53.805 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:53.805 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:53.805 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:53.805 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:53.844 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:53.844 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:53.844 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:53.844 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:53.844 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:53.845 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:53.845 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:53.845 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:53.845 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:53.846 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=201), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [19ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [161ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [161ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [161ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [161ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [199ms] refreshing cache from master, [199ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [200ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [201ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [201ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [201ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, deferred=Deferred@515307808(state=PENDING, result=null, callback=(continuation of Deferred@1235617162 after retry RPC after error@1913737850) -> (continuation of Deferred@1847302726 after retry RPC after error@1204617564) -> (continuation of Deferred@1044305530 after retry RPC after error@1178915141) -> (continuation of Deferred@1958718049 after retry RPC after error@1532465739) -> (continuation of Deferred@319947357 after retry RPC after error@1933326211) -> (continuation of Deferred@2061068071 after retry RPC after error@494630909) -> (continuation of Deferred@519605989 after retry RPC after error@208282262), errback=(continuation of Deferred@1235617162 after retry RPC after error@1913737850) -> (continuation of Deferred@1847302726 after retry RPC after error@1204617564) -> (continuation of Deferred@1044305530 after retry RPC after error@1178915141) -> (continuation of Deferred@1958718049 after retry RPC after error@1532465739) -> (continuation of Deferred@319947357 after retry RPC after error@1933326211) -> (continuation of Deferred@2061068071 after retry RPC after error@494630909) -> (continuation of Deferred@519605989 after retry RPC after error@208282262)))
21:04:53.846 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=201), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [19ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [161ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [161ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [161ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [161ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [199ms] refreshing cache from master, [199ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [200ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [201ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [201ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [201ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, deferred=Deferred@515307808(state=PENDING, result=null, callback=(continuation of Deferred@1235617162 after retry RPC after error@1913737850) -> (continuation of Deferred@1847302726 after retry RPC after error@1204617564) -> (continuation of Deferred@1044305530 after retry RPC after error@1178915141) -> (continuation of Deferred@1958718049 after retry RPC after error@1532465739) -> (continuation of Deferred@319947357 after retry RPC after error@1933326211) -> (continuation of Deferred@2061068071 after retry RPC after error@494630909) -> (continuation of Deferred@519605989 after retry RPC after error@208282262), errback=(continuation of Deferred@1235617162 after retry RPC after error@1913737850) -> (continuation of Deferred@1847302726 after retry RPC after error@1204617564) -> (continuation of Deferred@1044305530 after retry RPC after error@1178915141) -> (continuation of Deferred@1958718049 after retry RPC after error@1532465739) -> (continuation of Deferred@319947357 after retry RPC after error@1933326211) -> (continuation of Deferred@2061068071 after retry RPC after error@494630909) -> (continuation of Deferred@519605989 after retry RPC after error@208282262)))
21:04:53.864 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:53.865 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:53.865 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:53.865 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:53.865 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:53.865 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:53.865 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:53.866 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:53.866 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:53.884 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:53.884 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:53.885 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:53.885 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:53.885 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:53.885 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:53.885 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:53.885 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:53.886 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:53.904 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:53.904 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:53.905 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:53.905 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:53.905 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:53.905 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:53.905 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:53.905 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:53.906 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:53.924 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:53.924 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:53.924 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:53.924 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:53.925 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:53.925 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:53.925 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:53.925 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:53.925 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:53.944 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:53.945 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:53.945 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:53.945 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:53.945 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:53.945 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:53.946 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:53.946 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:53.946 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:53.984 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:53.985 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:53.985 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:53.985 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:53.985 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:53.985 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:53.985 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:53.986 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:53.986 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:54.044 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:54.045 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:54.045 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:54.045 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:54.045 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:54.045 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:54.046 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:54.046 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:54.046 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:54.064 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:54.064 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:54.065 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:54.065 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:54.065 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:54.065 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:54.065 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:54.065 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:54.066 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:54.084 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:54.084 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:54.085 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:54.085 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:54.085 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:54.085 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:54.085 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:54.085 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:54.086 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:54.104 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:54.104 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:54.105 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:54.105 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:54.105 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:54.105 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:54.105 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:54.105 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:54.106 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:54.106 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=10, TimeoutTracker(timeout=250, elapsed=241), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [59ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [61ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [181ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [181ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [182ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [201ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [201ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [202ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [221ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [221ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [222ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [222ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [241ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [241ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [242ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [242ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, deferred=Deferred@537426223(state=PENDING, result=null, callback=(continuation of Deferred@421812400 after retry RPC after error@1571853211) -> (continuation of Deferred@1191221934 after retry RPC after error@614617696) -> (continuation of Deferred@527986588 after retry RPC after error@1175260067) -> (continuation of Deferred@94059558 after retry RPC after error@865886119) -> (continuation of Deferred@1866781383 after retry RPC after error@1094356127) -> (continuation of Deferred@496820882 after retry RPC after error@1278011361) -> (continuation of Deferred@87541321 after retry RPC after error@276622340) -> (continuation of Deferred@1779307755 after retry RPC after error@1102509476) -> (continuation of Deferred@2047374933 after retry RPC after error@233028060), errback=(continuation of Deferred@421812400 after retry RPC after error@1571853211) -> (continuation of Deferred@1191221934 after retry RPC after error@614617696) -> (continuation of Deferred@527986588 after retry RPC after error@1175260067) -> (continuation of Deferred@94059558 after retry RPC after error@865886119) -> (continuation of Deferred@1866781383 after retry RPC after error@1094356127) -> (continuation of Deferred@496820882 after retry RPC after error@1278011361) -> (continuation of Deferred@87541321 after retry RPC after error@276622340) -> (continuation of Deferred@1779307755 after retry RPC after error@1102509476) -> (continuation of Deferred@2047374933 after retry RPC after error@233028060)))
21:04:54.107 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=10, TimeoutTracker(timeout=250, elapsed=241), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [59ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [61ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [181ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [181ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [182ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [201ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [201ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [202ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [221ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [221ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [222ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [222ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [241ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [241ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [242ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [242ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, deferred=Deferred@537426223(state=PENDING, result=null, callback=(continuation of Deferred@421812400 after retry RPC after error@1571853211) -> (continuation of Deferred@1191221934 after retry RPC after error@614617696) -> (continuation of Deferred@527986588 after retry RPC after error@1175260067) -> (continuation of Deferred@94059558 after retry RPC after error@865886119) -> (continuation of Deferred@1866781383 after retry RPC after error@1094356127) -> (continuation of Deferred@496820882 after retry RPC after error@1278011361) -> (continuation of Deferred@87541321 after retry RPC after error@276622340) -> (continuation of Deferred@1779307755 after retry RPC after error@1102509476) -> (continuation of Deferred@2047374933 after retry RPC after error@233028060), errback=(continuation of Deferred@421812400 after retry RPC after error@1571853211) -> (continuation of Deferred@1191221934 after retry RPC after error@614617696) -> (continuation of Deferred@527986588 after retry RPC after error@1175260067) -> (continuation of Deferred@94059558 after retry RPC after error@865886119) -> (continuation of Deferred@1866781383 after retry RPC after error@1094356127) -> (continuation of Deferred@496820882 after retry RPC after error@1278011361) -> (continuation of Deferred@87541321 after retry RPC after error@276622340) -> (continuation of Deferred@1779307755 after retry RPC after error@1102509476) -> (continuation of Deferred@2047374933 after retry RPC after error@233028060)))
21:04:54.124 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:54.124 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:54.125 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:54.125 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:54.125 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:54.125 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:54.125 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:54.125 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:54.126 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:54.144 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:54.145 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:54.145 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:54.145 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:54.145 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:54.145 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:54.145 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:54.145 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:54.146 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:54.164 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:54.164 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:54.165 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:54.165 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:54.165 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:54.165 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:54.165 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:54.165 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:54.166 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:54.184 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:54.184 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:54.185 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:54.185 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:54.185 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:54.185 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:54.185 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:54.185 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:54.186 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:54.204 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:54.204 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:54.205 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:54.205 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:54.205 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:54.205 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:54.205 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:54.205 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:54.206 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:54.244 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:54.244 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:54.245 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:54.245 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:54.245 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:54.245 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:54.245 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:54.245 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:54.246 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:54.304 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:54.304 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:54.305 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:54.305 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:54.305 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:54.305 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:54.305 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:54.305 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:54.306 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:54.306 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=182), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [3ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [21ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [22ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [22ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [23ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [41ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [43ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [61ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [63ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [81ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [83ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [120ms] refreshing cache from master, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [123ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [181ms] refreshing cache from master, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [182ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [182ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [183ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [183ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, deferred=Deferred@1817103233(state=PENDING, result=null, callback=(continuation of Deferred@743817284 after retry RPC after error@1286337334) -> (continuation of Deferred@2111390662 after retry RPC after error@1741786431) -> (continuation of Deferred@1694770785 after retry RPC after error@1990548404) -> (continuation of Deferred@1096151509 after retry RPC after error@713434293) -> (continuation of Deferred@1861560490 after retry RPC after error@844215867) -> (continuation of Deferred@1558459133 after retry RPC after error@1205605976), errback=(continuation of Deferred@743817284 after retry RPC after error@1286337334) -> (continuation of Deferred@2111390662 after retry RPC after error@1741786431) -> (continuation of Deferred@1694770785 after retry RPC after error@1990548404) -> (continuation of Deferred@1096151509 after retry RPC after error@713434293) -> (continuation of Deferred@1861560490 after retry RPC after error@844215867) -> (continuation of Deferred@1558459133 after retry RPC after error@1205605976)))
21:04:54.307 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=182), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [3ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [21ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [22ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [22ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [23ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [41ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [43ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [61ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [63ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [81ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [83ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [120ms] refreshing cache from master, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [123ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [181ms] refreshing cache from master, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [182ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [182ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [183ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [183ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, deferred=Deferred@1817103233(state=PENDING, result=null, callback=(continuation of Deferred@743817284 after retry RPC after error@1286337334) -> (continuation of Deferred@2111390662 after retry RPC after error@1741786431) -> (continuation of Deferred@1694770785 after retry RPC after error@1990548404) -> (continuation of Deferred@1096151509 after retry RPC after error@713434293) -> (continuation of Deferred@1861560490 after retry RPC after error@844215867) -> (continuation of Deferred@1558459133 after retry RPC after error@1205605976), errback=(continuation of Deferred@743817284 after retry RPC after error@1286337334) -> (continuation of Deferred@2111390662 after retry RPC after error@1741786431) -> (continuation of Deferred@1694770785 after retry RPC after error@1990548404) -> (continuation of Deferred@1096151509 after retry RPC after error@713434293) -> (continuation of Deferred@1861560490 after retry RPC after error@844215867) -> (continuation of Deferred@1558459133 after retry RPC after error@1205605976)))
21:04:54.325 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:54.325 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:54.325 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:54.326 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:54.326 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:54.326 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:54.326 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:54.326 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:54.327 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:54.344 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:54.345 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:54.345 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:54.345 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:54.345 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:54.345 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:54.345 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:54.345 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:54.346 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:54.364 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:54.365 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:54.365 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:54.365 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:54.365 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:54.365 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:54.365 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:54.365 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:54.366 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:54.384 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:54.384 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:54.385 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:54.385 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:54.385 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:54.385 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:54.385 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:54.385 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:54.386 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:54.404 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:54.404 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:54.405 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:54.405 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:54.405 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:54.405 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:54.405 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:54.405 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:54.406 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:54.424 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:54.425 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:54.425 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:54.425 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:54.425 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:54.425 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:54.426 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:54.426 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:54.426 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:54.484 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:54.485 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:54.485 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:54.485 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:54.486 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:54.486 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:54.486 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:54.486 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:54.490 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:54.564 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:54.564 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:54.565 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:54.565 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:54.565 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:54.565 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:54.565 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:54.565 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:54.565 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:54.566 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=241), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [162ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [162ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [165ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [166ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [239ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [241ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [241ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [241ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [242ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, deferred=Deferred@1468135432(state=PENDING, result=null, callback=(continuation of Deferred@2036302294 after retry RPC after error@1735948398) -> (continuation of Deferred@457081531 after retry RPC after error@1135044716) -> (continuation of Deferred@1190330623 after retry RPC after error@1651292999) -> (continuation of Deferred@220776171 after retry RPC after error@1974741802) -> (continuation of Deferred@731785627 after retry RPC after error@741280044) -> (continuation of Deferred@1736154588 after retry RPC after error@859969378) -> (continuation of Deferred@1774858848 after retry RPC after error@1031286812), errback=(continuation of Deferred@2036302294 after retry RPC after error@1735948398) -> (continuation of Deferred@457081531 after retry RPC after error@1135044716) -> (continuation of Deferred@1190330623 after retry RPC after error@1651292999) -> (continuation of Deferred@220776171 after retry RPC after error@1974741802) -> (continuation of Deferred@731785627 after retry RPC after error@741280044) -> (continuation of Deferred@1736154588 after retry RPC after error@859969378) -> (continuation of Deferred@1774858848 after retry RPC after error@1031286812)))
21:04:54.566 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=241), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [162ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [162ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [165ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [166ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [239ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [241ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [241ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [241ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [242ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, deferred=Deferred@1468135432(state=PENDING, result=null, callback=(continuation of Deferred@2036302294 after retry RPC after error@1735948398) -> (continuation of Deferred@457081531 after retry RPC after error@1135044716) -> (continuation of Deferred@1190330623 after retry RPC after error@1651292999) -> (continuation of Deferred@220776171 after retry RPC after error@1974741802) -> (continuation of Deferred@731785627 after retry RPC after error@741280044) -> (continuation of Deferred@1736154588 after retry RPC after error@859969378) -> (continuation of Deferred@1774858848 after retry RPC after error@1031286812), errback=(continuation of Deferred@2036302294 after retry RPC after error@1735948398) -> (continuation of Deferred@457081531 after retry RPC after error@1135044716) -> (continuation of Deferred@1190330623 after retry RPC after error@1651292999) -> (continuation of Deferred@220776171 after retry RPC after error@1974741802) -> (continuation of Deferred@731785627 after retry RPC after error@741280044) -> (continuation of Deferred@1736154588 after retry RPC after error@859969378) -> (continuation of Deferred@1774858848 after retry RPC after error@1031286812)))
21:04:54.584 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:54.585 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:54.585 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:54.585 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:54.585 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:54.585 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:54.586 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:54.586 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:54.586 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:54.604 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:54.604 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:54.605 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:54.605 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:54.605 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:54.605 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:54.605 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:54.606 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:54.606 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:54.624 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:54.624 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:54.625 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:54.625 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:54.625 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:54.625 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:54.625 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:54.625 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:54.626 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:54.644 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:54.644 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:54.644 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:54.644 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:54.645 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:54.645 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:54.645 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:54.645 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:54.646 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:54.664 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:54.665 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:54.665 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:54.665 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:54.665 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:54.665 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:54.665 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:54.665 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:54.666 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:54.684 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:54.684 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:54.685 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:54.685 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:54.685 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:54.685 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:54.685 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:54.685 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:54.686 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:54.744 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:54.744 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:54.745 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:54.745 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:54.745 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:54.745 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:54.745 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:54.746 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:54.746 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:54.746 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:54.746 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:54.747 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:54.764 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:54.764 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:54.765 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:54.765 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:54.765 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:54.765 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:54.765 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:54.765 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:54.766 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:54.767 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=181), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [59ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [161ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [162ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [162ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [163ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [181ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [181ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [182ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, deferred=Deferred@631780705(state=PENDING, result=null, callback=(continuation of Deferred@1946912905 after retry RPC after error@797831705) -> (continuation of Deferred@1285653627 after retry RPC after error@684110771) -> (continuation of Deferred@10566114 after retry RPC after error@1090264789) -> (continuation of Deferred@4114669 after retry RPC after error@494450802) -> (continuation of Deferred@337640470 after retry RPC after error@884294390) -> (continuation of Deferred@1111133485 after retry RPC after error@442046000) -> (continuation of Deferred@1900918305 after retry RPC after error@1878124556), errback=(continuation of Deferred@1946912905 after retry RPC after error@797831705) -> (continuation of Deferred@1285653627 after retry RPC after error@684110771) -> (continuation of Deferred@10566114 after retry RPC after error@1090264789) -> (continuation of Deferred@4114669 after retry RPC after error@494450802) -> (continuation of Deferred@337640470 after retry RPC after error@884294390) -> (continuation of Deferred@1111133485 after retry RPC after error@442046000) -> (continuation of Deferred@1900918305 after retry RPC after error@1878124556)))
21:04:54.767 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=181), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [59ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [161ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [162ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [162ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [163ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [181ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [181ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [182ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, deferred=Deferred@631780705(state=PENDING, result=null, callback=(continuation of Deferred@1946912905 after retry RPC after error@797831705) -> (continuation of Deferred@1285653627 after retry RPC after error@684110771) -> (continuation of Deferred@10566114 after retry RPC after error@1090264789) -> (continuation of Deferred@4114669 after retry RPC after error@494450802) -> (continuation of Deferred@337640470 after retry RPC after error@884294390) -> (continuation of Deferred@1111133485 after retry RPC after error@442046000) -> (continuation of Deferred@1900918305 after retry RPC after error@1878124556), errback=(continuation of Deferred@1946912905 after retry RPC after error@797831705) -> (continuation of Deferred@1285653627 after retry RPC after error@684110771) -> (continuation of Deferred@10566114 after retry RPC after error@1090264789) -> (continuation of Deferred@4114669 after retry RPC after error@494450802) -> (continuation of Deferred@337640470 after retry RPC after error@884294390) -> (continuation of Deferred@1111133485 after retry RPC after error@442046000) -> (continuation of Deferred@1900918305 after retry RPC after error@1878124556)))
21:04:54.784 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:54.784 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:54.784 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:54.784 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:54.784 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:54.784 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:54.785 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:54.785 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:54.786 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:54.804 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:54.805 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:54.805 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:54.805 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:54.805 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:54.805 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:54.805 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:54.805 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:54.806 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:54.824 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:54.825 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:54.825 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:54.825 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:54.825 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:54.825 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:54.825 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:54.826 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:54.827 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:54.844 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:54.844 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:54.845 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:54.845 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:54.845 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:54.845 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:54.845 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:54.845 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:54.846 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:54.864 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:54.864 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:54.865 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:54.865 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:54.865 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:54.865 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:54.865 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:54.865 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:54.866 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:54.884 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:54.884 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:54.884 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:54.884 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:54.884 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:54.885 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:54.885 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:54.885 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:54.885 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:54.904 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:54.905 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:54.905 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:54.905 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:54.905 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:54.905 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:54.905 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:54.906 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:54.906 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:54.944 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:54.945 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:54.945 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:54.945 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:54.945 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:54.945 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:54.945 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:54.945 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:54.946 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:54.947 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=163), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [21ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [23ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [41ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [44ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [44ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [61ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [63ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [81ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [83ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [121ms] refreshing cache from master, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [122ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [122ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [123ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [161ms] refreshing cache from master, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [162ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [162ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [163ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [164ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, deferred=Deferred@2066980829(state=PENDING, result=null, callback=(continuation of Deferred@711247447 after retry RPC after error@994215263) -> (continuation of Deferred@1032677565 after retry RPC after error@1512560038) -> (continuation of Deferred@1459484363 after retry RPC after error@2033908022) -> (continuation of Deferred@799325740 after retry RPC after error@1263195291) -> (continuation of Deferred@1916287127 after retry RPC after error@1550159023) -> (continuation of Deferred@1446649041 after retry RPC after error@1271522433) -> (continuation of Deferred@2022081007 after retry RPC after error@141982372), errback=(continuation of Deferred@711247447 after retry RPC after error@994215263) -> (continuation of Deferred@1032677565 after retry RPC after error@1512560038) -> (continuation of Deferred@1459484363 after retry RPC after error@2033908022) -> (continuation of Deferred@799325740 after retry RPC after error@1263195291) -> (continuation of Deferred@1916287127 after retry RPC after error@1550159023) -> (continuation of Deferred@1446649041 after retry RPC after error@1271522433) -> (continuation of Deferred@2022081007 after retry RPC after error@141982372)))
21:04:54.948 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=163), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [21ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [23ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [41ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [44ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [44ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [61ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [63ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [81ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [83ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [121ms] refreshing cache from master, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [122ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [122ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [123ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [161ms] refreshing cache from master, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [162ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [162ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [163ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [164ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, deferred=Deferred@2066980829(state=PENDING, result=null, callback=(continuation of Deferred@711247447 after retry RPC after error@994215263) -> (continuation of Deferred@1032677565 after retry RPC after error@1512560038) -> (continuation of Deferred@1459484363 after retry RPC after error@2033908022) -> (continuation of Deferred@799325740 after retry RPC after error@1263195291) -> (continuation of Deferred@1916287127 after retry RPC after error@1550159023) -> (continuation of Deferred@1446649041 after retry RPC after error@1271522433) -> (continuation of Deferred@2022081007 after retry RPC after error@141982372), errback=(continuation of Deferred@711247447 after retry RPC after error@994215263) -> (continuation of Deferred@1032677565 after retry RPC after error@1512560038) -> (continuation of Deferred@1459484363 after retry RPC after error@2033908022) -> (continuation of Deferred@799325740 after retry RPC after error@1263195291) -> (continuation of Deferred@1916287127 after retry RPC after error@1550159023) -> (continuation of Deferred@1446649041 after retry RPC after error@1271522433) -> (continuation of Deferred@2022081007 after retry RPC after error@141982372)))
21:04:54.964 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:54.964 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:54.965 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:54.965 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:54.965 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:54.965 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:54.965 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:54.965 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:54.966 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:54.984 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:54.984 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:54.985 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:54.985 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:54.985 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:54.985 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:54.985 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:54.985 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:54.986 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:55.004 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:55.004 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:55.005 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:55.005 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:55.005 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:55.005 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:55.005 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:55.005 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:55.006 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:55.024 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:55.024 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:55.024 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:55.024 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:55.024 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:55.024 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:55.025 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:55.025 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:55.026 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:55.044 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:55.045 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:55.045 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:55.045 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:55.045 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:55.045 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:55.045 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:55.046 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:55.045 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:55.046 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:55.046 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:55.046 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:55.084 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:55.084 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:55.085 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:55.085 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:55.085 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:55.085 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:55.085 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:55.085 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:55.085 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:55.085 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:55.086 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:55.086 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:55.144 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:55.145 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:55.145 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:55.145 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:55.145 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:55.145 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:55.145 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:55.145 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:55.146 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:55.184 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:55.185 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:55.185 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:55.185 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:55.185 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:55.185 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:55.185 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:55.185 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:55.186 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:55.187 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=222), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [59ms] refreshing cache from master, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [60ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [81ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [121ms] refreshing cache from master, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [181ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [181ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [182ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [221ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [221ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [222ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [222ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, deferred=Deferred@1475044905(state=PENDING, result=null, callback=(continuation of Deferred@1418701869 after retry RPC after error@861212385) -> (continuation of Deferred@1896831201 after retry RPC after error@2111419079) -> (continuation of Deferred@1419675541 after retry RPC after error@1336388065) -> (continuation of Deferred@1155807396 after retry RPC after error@1956373877) -> (continuation of Deferred@1419343452 after retry RPC after error@637744527) -> (continuation of Deferred@1137117427 after retry RPC after error@1750920627) -> (continuation of Deferred@2011712882 after retry RPC after error@1215865165), errback=(continuation of Deferred@1418701869 after retry RPC after error@861212385) -> (continuation of Deferred@1896831201 after retry RPC after error@2111419079) -> (continuation of Deferred@1419675541 after retry RPC after error@1336388065) -> (continuation of Deferred@1155807396 after retry RPC after error@1956373877) -> (continuation of Deferred@1419343452 after retry RPC after error@637744527) -> (continuation of Deferred@1137117427 after retry RPC after error@1750920627) -> (continuation of Deferred@2011712882 after retry RPC after error@1215865165)))
21:04:55.187 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=222), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [59ms] refreshing cache from master, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [60ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [81ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [121ms] refreshing cache from master, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [181ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [181ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [182ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [221ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [221ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [222ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [222ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, deferred=Deferred@1475044905(state=PENDING, result=null, callback=(continuation of Deferred@1418701869 after retry RPC after error@861212385) -> (continuation of Deferred@1896831201 after retry RPC after error@2111419079) -> (continuation of Deferred@1419675541 after retry RPC after error@1336388065) -> (continuation of Deferred@1155807396 after retry RPC after error@1956373877) -> (continuation of Deferred@1419343452 after retry RPC after error@637744527) -> (continuation of Deferred@1137117427 after retry RPC after error@1750920627) -> (continuation of Deferred@2011712882 after retry RPC after error@1215865165), errback=(continuation of Deferred@1418701869 after retry RPC after error@861212385) -> (continuation of Deferred@1896831201 after retry RPC after error@2111419079) -> (continuation of Deferred@1419675541 after retry RPC after error@1336388065) -> (continuation of Deferred@1155807396 after retry RPC after error@1956373877) -> (continuation of Deferred@1419343452 after retry RPC after error@637744527) -> (continuation of Deferred@1137117427 after retry RPC after error@1750920627) -> (continuation of Deferred@2011712882 after retry RPC after error@1215865165)))
21:04:55.204 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:55.204 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:55.205 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:55.205 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:55.205 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:55.205 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:55.205 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:55.205 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:55.206 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:55.224 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:55.224 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:55.225 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:55.225 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:55.225 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:55.225 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:55.225 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:55.225 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:55.226 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:55.244 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:55.244 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:55.244 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:55.244 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:55.245 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:55.245 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:55.245 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:55.245 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:55.246 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:55.264 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:55.265 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:55.265 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:55.265 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:55.265 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:55.265 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:55.265 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:55.265 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:55.266 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:55.284 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:55.284 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:55.285 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:55.285 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:55.285 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:55.285 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:55.285 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:55.285 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:55.286 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:55.304 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:55.304 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:55.305 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:55.305 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:55.305 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:55.305 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:55.305 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:55.305 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:55.306 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:55.344 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:55.344 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:55.345 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:55.345 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:55.345 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:55.345 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:55.345 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:55.345 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:55.346 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:55.364 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:55.364 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:55.365 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:55.365 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:55.365 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:55.365 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:55.365 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:55.365 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:55.366 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:55.444 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:55.444 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:55.445 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:55.445 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:55.445 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:55.445 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:55.445 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:55.445 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:55.446 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:55.446 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=9, TimeoutTracker(timeout=250, elapsed=242), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [39ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [141ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [141ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [142ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [159ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [161ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [161ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [162ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [241ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [241ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [242ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [242ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, deferred=Deferred@517830174(state=PENDING, result=null, callback=(continuation of Deferred@1915433748 after retry RPC after error@1438817543) -> (continuation of Deferred@1333162950 after retry RPC after error@988713392) -> (continuation of Deferred@292988522 after retry RPC after error@1352481766) -> (continuation of Deferred@747215472 after retry RPC after error@1646591296) -> (continuation of Deferred@1178944152 after retry RPC after error@2143159229) -> (continuation of Deferred@520379424 after retry RPC after error@1519348861) -> (continuation of Deferred@917681206 after retry RPC after error@958331574) -> (continuation of Deferred@80102633 after retry RPC after error@621489288), errback=(continuation of Deferred@1915433748 after retry RPC after error@1438817543) -> (continuation of Deferred@1333162950 after retry RPC after error@988713392) -> (continuation of Deferred@292988522 after retry RPC after error@1352481766) -> (continuation of Deferred@747215472 after retry RPC after error@1646591296) -> (continuation of Deferred@1178944152 after retry RPC after error@2143159229) -> (continuation of Deferred@520379424 after retry RPC after error@1519348861) -> (continuation of Deferred@917681206 after retry RPC after error@958331574) -> (continuation of Deferred@80102633 after retry RPC after error@621489288)))
21:04:55.447 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=9, TimeoutTracker(timeout=250, elapsed=242), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [39ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [141ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [141ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [142ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [159ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [161ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [161ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [162ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [241ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [241ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [242ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [242ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, deferred=Deferred@517830174(state=PENDING, result=null, callback=(continuation of Deferred@1915433748 after retry RPC after error@1438817543) -> (continuation of Deferred@1333162950 after retry RPC after error@988713392) -> (continuation of Deferred@292988522 after retry RPC after error@1352481766) -> (continuation of Deferred@747215472 after retry RPC after error@1646591296) -> (continuation of Deferred@1178944152 after retry RPC after error@2143159229) -> (continuation of Deferred@520379424 after retry RPC after error@1519348861) -> (continuation of Deferred@917681206 after retry RPC after error@958331574) -> (continuation of Deferred@80102633 after retry RPC after error@621489288), errback=(continuation of Deferred@1915433748 after retry RPC after error@1438817543) -> (continuation of Deferred@1333162950 after retry RPC after error@988713392) -> (continuation of Deferred@292988522 after retry RPC after error@1352481766) -> (continuation of Deferred@747215472 after retry RPC after error@1646591296) -> (continuation of Deferred@1178944152 after retry RPC after error@2143159229) -> (continuation of Deferred@520379424 after retry RPC after error@1519348861) -> (continuation of Deferred@917681206 after retry RPC after error@958331574) -> (continuation of Deferred@80102633 after retry RPC after error@621489288)))
21:04:55.464 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:55.464 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:55.465 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:55.465 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:55.465 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:55.465 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:55.465 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:55.465 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:55.466 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:55.485 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:55.485 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:55.485 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:55.485 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:55.485 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:55.486 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:55.486 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:55.486 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:55.487 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:55.504 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:55.505 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:55.505 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:55.505 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:55.505 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:55.505 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:55.505 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:55.505 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:55.506 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:55.524 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:55.524 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:55.525 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:55.525 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:55.525 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:55.525 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:55.525 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:55.525 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:55.526 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:55.544 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:55.544 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:55.545 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:55.545 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:55.545 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:55.545 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:55.545 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:55.545 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:55.546 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:55.564 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:55.564 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:55.564 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:55.564 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:55.564 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:55.565 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:55.565 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:55.565 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:55.566 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:55.584 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:55.585 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:55.585 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:55.585 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:55.585 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:55.585 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:55.585 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:55.585 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:55.586 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:55.684 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:55.684 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:55.685 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:55.685 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:55.685 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:55.685 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:55.685 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:55.685 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:55.685 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:55.686 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=221), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [99ms] refreshing cache from master, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [219ms] refreshing cache from master, [219ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [221ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [221ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [221ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [222ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, deferred=Deferred@452417903(state=PENDING, result=null, callback=(continuation of Deferred@874206674 after retry RPC after error@545313897) -> (continuation of Deferred@683962012 after retry RPC after error@140015306) -> (continuation of Deferred@1719867350 after retry RPC after error@1082569971) -> (continuation of Deferred@1262326513 after retry RPC after error@9978746) -> (continuation of Deferred@2119944472 after retry RPC after error@193973004) -> (continuation of Deferred@738355862 after retry RPC after error@1858301951) -> (continuation of Deferred@1315989558 after retry RPC after error@995237215), errback=(continuation of Deferred@874206674 after retry RPC after error@545313897) -> (continuation of Deferred@683962012 after retry RPC after error@140015306) -> (continuation of Deferred@1719867350 after retry RPC after error@1082569971) -> (continuation of Deferred@1262326513 after retry RPC after error@9978746) -> (continuation of Deferred@2119944472 after retry RPC after error@193973004) -> (continuation of Deferred@738355862 after retry RPC after error@1858301951) -> (continuation of Deferred@1315989558 after retry RPC after error@995237215)))
21:04:55.686 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=221), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [99ms] refreshing cache from master, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [219ms] refreshing cache from master, [219ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [221ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [221ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [221ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [222ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, deferred=Deferred@452417903(state=PENDING, result=null, callback=(continuation of Deferred@874206674 after retry RPC after error@545313897) -> (continuation of Deferred@683962012 after retry RPC after error@140015306) -> (continuation of Deferred@1719867350 after retry RPC after error@1082569971) -> (continuation of Deferred@1262326513 after retry RPC after error@9978746) -> (continuation of Deferred@2119944472 after retry RPC after error@193973004) -> (continuation of Deferred@738355862 after retry RPC after error@1858301951) -> (continuation of Deferred@1315989558 after retry RPC after error@995237215), errback=(continuation of Deferred@874206674 after retry RPC after error@545313897) -> (continuation of Deferred@683962012 after retry RPC after error@140015306) -> (continuation of Deferred@1719867350 after retry RPC after error@1082569971) -> (continuation of Deferred@1262326513 after retry RPC after error@9978746) -> (continuation of Deferred@2119944472 after retry RPC after error@193973004) -> (continuation of Deferred@738355862 after retry RPC after error@1858301951) -> (continuation of Deferred@1315989558 after retry RPC after error@995237215)))
21:04:55.704 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:55.705 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:55.705 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:55.705 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:55.705 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:55.705 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:55.705 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:55.705 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:55.706 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:55.724 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:55.724 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:55.725 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:55.725 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:55.725 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:55.725 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:55.725 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:55.725 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:55.726 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:55.744 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:55.744 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:55.745 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:55.745 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:55.745 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:55.745 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:55.745 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:55.745 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:55.746 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:55.764 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:55.765 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:55.765 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:55.765 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:55.765 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:55.766 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:55.766 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:55.766 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:55.767 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:55.784 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:55.784 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:55.785 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:55.785 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:55.785 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:55.785 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:55.785 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:55.785 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:55.786 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:55.804 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:55.804 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:55.805 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:55.805 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:55.805 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:55.805 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:55.805 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:55.805 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:55.806 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:55.864 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:55.864 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:55.865 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:55.865 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:55.865 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:55.865 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:55.865 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:55.865 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:55.866 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:55.867 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=162), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [63ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [161ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [161ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [162ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, deferred=Deferred@1566532845(state=PENDING, result=null, callback=(continuation of Deferred@1208538719 after retry RPC after error@1900832125) -> (continuation of Deferred@1705028381 after retry RPC after error@1132385096) -> (continuation of Deferred@1296719702 after retry RPC after error@1598012431) -> (continuation of Deferred@1067912374 after retry RPC after error@175537305) -> (continuation of Deferred@366551662 after retry RPC after error@933509918) -> (continuation of Deferred@811684832 after retry RPC after error@155870145), errback=(continuation of Deferred@1208538719 after retry RPC after error@1900832125) -> (continuation of Deferred@1705028381 after retry RPC after error@1132385096) -> (continuation of Deferred@1296719702 after retry RPC after error@1598012431) -> (continuation of Deferred@1067912374 after retry RPC after error@175537305) -> (continuation of Deferred@366551662 after retry RPC after error@933509918) -> (continuation of Deferred@811684832 after retry RPC after error@155870145)))
21:04:55.867 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=162), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [63ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [161ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [161ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [162ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, deferred=Deferred@1566532845(state=PENDING, result=null, callback=(continuation of Deferred@1208538719 after retry RPC after error@1900832125) -> (continuation of Deferred@1705028381 after retry RPC after error@1132385096) -> (continuation of Deferred@1296719702 after retry RPC after error@1598012431) -> (continuation of Deferred@1067912374 after retry RPC after error@175537305) -> (continuation of Deferred@366551662 after retry RPC after error@933509918) -> (continuation of Deferred@811684832 after retry RPC after error@155870145), errback=(continuation of Deferred@1208538719 after retry RPC after error@1900832125) -> (continuation of Deferred@1705028381 after retry RPC after error@1132385096) -> (continuation of Deferred@1296719702 after retry RPC after error@1598012431) -> (continuation of Deferred@1067912374 after retry RPC after error@175537305) -> (continuation of Deferred@366551662 after retry RPC after error@933509918) -> (continuation of Deferred@811684832 after retry RPC after error@155870145)))
21:04:55.884 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:55.884 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:55.885 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:55.885 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:55.885 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:55.885 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:55.885 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:55.885 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:55.886 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:55.905 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:55.906 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:55.906 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:55.906 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:55.906 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:55.907 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:55.907 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:55.907 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:55.907 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:55.924 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:55.924 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:55.925 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:55.925 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:55.925 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:55.925 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:55.925 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:55.925 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:55.926 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:55.944 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:55.944 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:55.945 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:55.945 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:55.945 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:55.945 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:55.945 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:55.945 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:55.946 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:55.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:55.950747 24071 master_runner.cc:186] Error getting master registration for 127.22.168.62:37239: Network error: Client connection negotiation failed: client connection to 127.22.168.62:37239: connect: Connection refused (error 111),
21:04:55.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:55.950964 24071 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
21:04:55.964 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:55.964 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:55.964 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:55.964 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:55.964 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:55.964 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:55.965 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:55.965 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:55.965 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:56.004 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:56.005 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:56.005 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:56.005 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:56.005 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:56.005 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:56.005 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:56.006 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:56.006 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:56.044 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:56.044 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:56.045 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:56.045 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:56.045 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:56.045 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:56.045 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:56.045 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:56.046 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:56.064 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:56.064 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:56.065 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:56.065 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:56.065 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:56.065 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:56.065 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:56.065 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:56.066 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:56.124 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:56.124 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:56.125 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:56.125 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:56.125 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:56.125 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:56.125 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:56.125 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:56.126 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:56.126 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=9, TimeoutTracker(timeout=250, elapsed=242), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [23ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [23ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [23ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [79ms] refreshing cache from master, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [81ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [161ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [161ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [162ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [181ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [181ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [182ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [241ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [241ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [242ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [242ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, deferred=Deferred@810448374(state=PENDING, result=null, callback=(continuation of Deferred@998450007 after retry RPC after error@983421762) -> (continuation of Deferred@10800399 after retry RPC after error@1132919080) -> (continuation of Deferred@1631450004 after retry RPC after error@904805170) -> (continuation of Deferred@327439413 after retry RPC after error@1740611796) -> (continuation of Deferred@1771069156 after retry RPC after error@761519951) -> (continuation of Deferred@477561642 after retry RPC after error@106215792) -> (continuation of Deferred@1834528800 after retry RPC after error@1592955747) -> (continuation of Deferred@2050536900 after retry RPC after error@1501830346), errback=(continuation of Deferred@998450007 after retry RPC after error@983421762) -> (continuation of Deferred@10800399 after retry RPC after error@1132919080) -> (continuation of Deferred@1631450004 after retry RPC after error@904805170) -> (continuation of Deferred@327439413 after retry RPC after error@1740611796) -> (continuation of Deferred@1771069156 after retry RPC after error@761519951) -> (continuation of Deferred@477561642 after retry RPC after error@106215792) -> (continuation of Deferred@1834528800 after retry RPC after error@1592955747) -> (continuation of Deferred@2050536900 after retry RPC after error@1501830346)))
21:04:56.127 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=9, TimeoutTracker(timeout=250, elapsed=242), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [23ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [23ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [23ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [79ms] refreshing cache from master, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [81ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [161ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [161ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [162ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [181ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [181ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [182ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [241ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [241ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [242ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [242ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, deferred=Deferred@810448374(state=PENDING, result=null, callback=(continuation of Deferred@998450007 after retry RPC after error@983421762) -> (continuation of Deferred@10800399 after retry RPC after error@1132919080) -> (continuation of Deferred@1631450004 after retry RPC after error@904805170) -> (continuation of Deferred@327439413 after retry RPC after error@1740611796) -> (continuation of Deferred@1771069156 after retry RPC after error@761519951) -> (continuation of Deferred@477561642 after retry RPC after error@106215792) -> (continuation of Deferred@1834528800 after retry RPC after error@1592955747) -> (continuation of Deferred@2050536900 after retry RPC after error@1501830346), errback=(continuation of Deferred@998450007 after retry RPC after error@983421762) -> (continuation of Deferred@10800399 after retry RPC after error@1132919080) -> (continuation of Deferred@1631450004 after retry RPC after error@904805170) -> (continuation of Deferred@327439413 after retry RPC after error@1740611796) -> (continuation of Deferred@1771069156 after retry RPC after error@761519951) -> (continuation of Deferred@477561642 after retry RPC after error@106215792) -> (continuation of Deferred@1834528800 after retry RPC after error@1592955747) -> (continuation of Deferred@2050536900 after retry RPC after error@1501830346)))
21:04:56.144 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:56.144 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:56.144 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:56.144 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:56.145 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:56.145 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:56.145 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:56.145 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:56.145 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:56.164 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:56.164 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:56.165 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:56.165 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:56.165 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:56.165 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:56.165 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:56.165 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:56.166 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:56.184 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:56.184 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:56.185 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:56.185 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:56.186 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:56.186 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:56.186 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:56.186 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:56.186 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:56.204 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:56.204 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:56.204 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:56.205 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:56.205 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:56.205 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:56.205 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:56.205 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:56.205 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:56.224 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:56.225 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:56.225 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:56.225 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:56.225 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:56.225 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:56.226 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:56.226 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:56.227 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:56.264 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:56.265 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:56.265 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:56.265 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:56.265 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:56.265 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:56.265 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:56.266 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:56.266 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:56.304 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:56.304 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:56.305 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:56.305 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:56.305 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:56.305 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:56.305 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:56.305 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:56.306 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:56.324 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:56.324 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:56.325 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:56.325 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:56.325 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:56.325 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:56.325 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:56.325 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:56.326 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:56.327 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=182), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [21ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [23ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [41ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [43ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [43ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [43ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [60ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [81ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [82ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [82ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [83ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [83ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [84ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [121ms] refreshing cache from master, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [122ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [122ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [123ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [123ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [161ms] refreshing cache from master, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [162ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [162ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [163ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [163ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [181ms] refreshing cache from master, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [182ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [182ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [183ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [183ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, deferred=Deferred@1316908312(state=PENDING, result=null, callback=(continuation of Deferred@1504478115 after retry RPC after error@41355808) -> (continuation of Deferred@1497575720 after retry RPC after error@1684574645) -> (continuation of Deferred@1499768576 after retry RPC after error@1668841478) -> (continuation of Deferred@594687792 after retry RPC after error@54001618) -> (continuation of Deferred@2132214803 after retry RPC after error@1489552199) -> (continuation of Deferred@1781638361 after retry RPC after error@1447304994) -> (continuation of Deferred@297057893 after retry RPC after error@267945455), errback=(continuation of Deferred@1504478115 after retry RPC after error@41355808) -> (continuation of Deferred@1497575720 after retry RPC after error@1684574645) -> (continuation of Deferred@1499768576 after retry RPC after error@1668841478) -> (continuation of Deferred@594687792 after retry RPC after error@54001618) -> (continuation of Deferred@2132214803 after retry RPC after error@1489552199) -> (continuation of Deferred@1781638361 after retry RPC after error@1447304994) -> (continuation of Deferred@297057893 after retry RPC after error@267945455)))
21:04:56.333 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=182), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [21ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [23ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [41ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [43ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [43ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [43ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [60ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [81ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [82ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [82ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [83ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [83ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [84ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [121ms] refreshing cache from master, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [122ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [122ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [123ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [123ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [161ms] refreshing cache from master, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [162ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [162ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [163ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [163ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [181ms] refreshing cache from master, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [182ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [182ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [183ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [183ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, deferred=Deferred@1316908312(state=PENDING, result=null, callback=(continuation of Deferred@1504478115 after retry RPC after error@41355808) -> (continuation of Deferred@1497575720 after retry RPC after error@1684574645) -> (continuation of Deferred@1499768576 after retry RPC after error@1668841478) -> (continuation of Deferred@594687792 after retry RPC after error@54001618) -> (continuation of Deferred@2132214803 after retry RPC after error@1489552199) -> (continuation of Deferred@1781638361 after retry RPC after error@1447304994) -> (continuation of Deferred@297057893 after retry RPC after error@267945455), errback=(continuation of Deferred@1504478115 after retry RPC after error@41355808) -> (continuation of Deferred@1497575720 after retry RPC after error@1684574645) -> (continuation of Deferred@1499768576 after retry RPC after error@1668841478) -> (continuation of Deferred@594687792 after retry RPC after error@54001618) -> (continuation of Deferred@2132214803 after retry RPC after error@1489552199) -> (continuation of Deferred@1781638361 after retry RPC after error@1447304994) -> (continuation of Deferred@297057893 after retry RPC after error@267945455)))
21:04:56.344 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:56.344 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:56.345 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:56.345 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:56.345 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:56.345 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:56.345 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:56.345 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:56.346 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:56.364 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:56.364 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:56.364 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:56.364 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:56.364 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:56.365 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:56.365 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:56.365 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:56.365 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:56.384 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:56.385 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:56.385 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:56.385 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:56.385 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:56.386 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:56.386 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:56.386 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:56.387 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:56.404 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:56.404 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:56.405 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:56.405 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:56.405 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:56.405 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:56.405 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:56.405 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:56.406 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:56.424 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:56.424 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:56.425 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:56.425 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:56.425 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:56.425 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:56.425 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:56.425 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:56.426 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:56.444 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:56.444 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:56.444 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:56.444 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:56.444 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:56.445 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:56.445 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:56.445 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:56.446 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:56.484 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:56.485 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:56.485 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:56.485 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:56.485 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:56.485 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:56.485 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:56.486 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:56.486 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:56.524 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:56.524 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:56.525 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:56.525 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:56.525 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:56.525 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:56.525 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:56.525 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:56.526 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:56.527 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=182), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [19ms] refreshing cache from master, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [43ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [99ms] refreshing cache from master, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [100ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [141ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [141ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [142ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [181ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [181ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [182ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, deferred=Deferred@1169101695(state=PENDING, result=null, callback=(continuation of Deferred@153172544 after retry RPC after error@2109149233) -> (continuation of Deferred@1829179738 after retry RPC after error@1351537518) -> (continuation of Deferred@1358761989 after retry RPC after error@361669811) -> (continuation of Deferred@1087830681 after retry RPC after error@1726301262) -> (continuation of Deferred@1587547567 after retry RPC after error@1055625716) -> (continuation of Deferred@1226749570 after retry RPC after error@93670619) -> (continuation of Deferred@1527182515 after retry RPC after error@964733462), errback=(continuation of Deferred@153172544 after retry RPC after error@2109149233) -> (continuation of Deferred@1829179738 after retry RPC after error@1351537518) -> (continuation of Deferred@1358761989 after retry RPC after error@361669811) -> (continuation of Deferred@1087830681 after retry RPC after error@1726301262) -> (continuation of Deferred@1587547567 after retry RPC after error@1055625716) -> (continuation of Deferred@1226749570 after retry RPC after error@93670619) -> (continuation of Deferred@1527182515 after retry RPC after error@964733462)))
21:04:56.527 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=182), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [19ms] refreshing cache from master, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [43ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [99ms] refreshing cache from master, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [100ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [141ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [141ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [142ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [181ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [181ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [182ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, deferred=Deferred@1169101695(state=PENDING, result=null, callback=(continuation of Deferred@153172544 after retry RPC after error@2109149233) -> (continuation of Deferred@1829179738 after retry RPC after error@1351537518) -> (continuation of Deferred@1358761989 after retry RPC after error@361669811) -> (continuation of Deferred@1087830681 after retry RPC after error@1726301262) -> (continuation of Deferred@1587547567 after retry RPC after error@1055625716) -> (continuation of Deferred@1226749570 after retry RPC after error@93670619) -> (continuation of Deferred@1527182515 after retry RPC after error@964733462), errback=(continuation of Deferred@153172544 after retry RPC after error@2109149233) -> (continuation of Deferred@1829179738 after retry RPC after error@1351537518) -> (continuation of Deferred@1358761989 after retry RPC after error@361669811) -> (continuation of Deferred@1087830681 after retry RPC after error@1726301262) -> (continuation of Deferred@1587547567 after retry RPC after error@1055625716) -> (continuation of Deferred@1226749570 after retry RPC after error@93670619) -> (continuation of Deferred@1527182515 after retry RPC after error@964733462)))
21:04:56.544 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:56.544 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:56.545 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:56.545 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:56.545 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:56.545 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:56.545 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:56.545 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:56.546 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:56.564 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:56.564 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:56.564 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:56.565 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:56.565 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:56.565 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:56.565 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:56.565 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:56.566 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:56.584 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:56.585 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:56.585 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:56.585 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:56.585 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:56.585 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:56.585 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:56.585 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:56.586 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:56.604 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:56.604 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:56.605 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:56.605 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:56.605 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:56.605 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:56.605 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:56.605 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:56.606 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:56.624 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:56.624 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:56.625 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:56.625 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:56.625 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:56.625 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:56.625 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:56.625 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:56.630 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:56.664 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:56.664 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:56.665 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:56.665 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:56.665 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:56.665 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:56.665 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:56.665 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:56.666 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:56.684 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:56.684 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:56.684 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:56.684 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:56.684 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:56.685 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:56.685 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:56.685 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:56.686 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:56.804 [DEBUG - pool-40-thread-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=260), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [86ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [86ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [139ms] refreshing cache from master, [139ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [140ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [141ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [141ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, deferred=Deferred@1582842012(state=PENDING, result=null, callback=(continuation of Deferred@2065411796 after retry RPC after error@1129388395) -> (continuation of Deferred@519939686 after retry RPC after error@45963653) -> (continuation of Deferred@620116488 after retry RPC after error@1864885692) -> (continuation of Deferred@471511208 after retry RPC after error@1954784223) -> (continuation of Deferred@1630597437 after retry RPC after error@829088418) -> (continuation of Deferred@2025626583 after retry RPC after error@289948228) -> (continuation of Deferred@2006446153 after retry RPC after error@286155755), errback=(continuation of Deferred@2065411796 after retry RPC after error@1129388395) -> (continuation of Deferred@519939686 after retry RPC after error@45963653) -> (continuation of Deferred@620116488 after retry RPC after error@1864885692) -> (continuation of Deferred@471511208 after retry RPC after error@1954784223) -> (continuation of Deferred@1630597437 after retry RPC after error@829088418) -> (continuation of Deferred@2025626583 after retry RPC after error@289948228) -> (continuation of Deferred@2006446153 after retry RPC after error@286155755)))
21:04:56.805 [WARN - pool-40-thread-1] (HashedWheelTimer.java:716) An exception was thrown by TimerTask.
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343) ~[main/:?]
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at java.lang.Thread.run(Thread.java:748) [?:1.8.0_275]
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86)
at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106)
at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198)
at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158)
at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953)
at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923)
at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464)
at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713)
at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701)
at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788)
at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501)
at java.lang.Thread.run(Thread.java:748)
21:04:56.824 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:56.824 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:56.824 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:56.825 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:56.825 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:56.825 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:56.825 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:56.825 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:56.826 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:56.844 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:56.845 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:56.845 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:56.845 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:56.845 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:56.845 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:56.846 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:56.846 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:56.847 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:56.864 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:56.865 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:56.865 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:56.865 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:56.865 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:56.865 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:56.866 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:56.866 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:56.866 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:56.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:56.879040 24510 raft_consensus.cc:493] T 00000000000000000000000000000000 P eb664add87f44013b596f4cc01c9a41c [term 4 FOLLOWER]: Starting pre-election (detected failure of leader c520077acf3341e29447c6c2b8d3f74c)
21:04:56.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:56.879302 24510 raft_consensus.cc:515] T 00000000000000000000000000000000 P eb664add87f44013b596f4cc01c9a41c [term 4 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e9dd218ca2244f5a934aac2b7e58eb1d" member_type: VOTER last_known_addr { host: "127.22.168.62" port: 37239 } } peers { permanent_uuid: "c520077acf3341e29447c6c2b8d3f74c" member_type: VOTER last_known_addr { host: "127.22.168.61" port: 32947 } } peers { permanent_uuid: "eb664add87f44013b596f4cc01c9a41c" member_type: VOTER last_known_addr { host: "127.22.168.60" port: 33247 } }
21:04:56.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:56.880097 24510 leader_election.cc:290] T 00000000000000000000000000000000 P eb664add87f44013b596f4cc01c9a41c [CANDIDATE]: Term 5 pre-election: Requested pre-vote from peers e9dd218ca2244f5a934aac2b7e58eb1d (127.22.168.62:37239), c520077acf3341e29447c6c2b8d3f74c (127.22.168.61:32947)
21:04:56.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:56.881199 24089 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.22.168.61:32947: connect: Connection refused (error 111) [suppressed 3 similar messages]
21:04:56.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:56.882614 24089 leader_election.cc:336] T 00000000000000000000000000000000 P eb664add87f44013b596f4cc01c9a41c [CANDIDATE]: Term 5 pre-election: RPC error from VoteRequest() call to peer c520077acf3341e29447c6c2b8d3f74c (127.22.168.61:32947): Network error: Client connection negotiation failed: client connection to 127.22.168.61:32947: connect: Connection refused (error 111)
21:04:56.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:56.882779 24086 leader_election.cc:336] T 00000000000000000000000000000000 P eb664add87f44013b596f4cc01c9a41c [CANDIDATE]: Term 5 pre-election: RPC error from VoteRequest() call to peer e9dd218ca2244f5a934aac2b7e58eb1d (127.22.168.62:37239): Network error: Client connection negotiation failed: client connection to 127.22.168.62:37239: connect: Connection refused (error 111)
21:04:56.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:56.882910 24086 leader_election.cc:304] T 00000000000000000000000000000000 P eb664add87f44013b596f4cc01c9a41c [CANDIDATE]: Term 5 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: eb664add87f44013b596f4cc01c9a41c; no voters: c520077acf3341e29447c6c2b8d3f74c, e9dd218ca2244f5a934aac2b7e58eb1d
21:04:56.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:56.883170 24510 raft_consensus.cc:2749] T 00000000000000000000000000000000 P eb664add87f44013b596f4cc01c9a41c [term 4 FOLLOWER]: Leader pre-election lost for term 5. Reason: could not achieve majority
21:04:56.884 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:56.884 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:56.885 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:56.885 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:56.885 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:56.885 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:56.885 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:56.885 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:56.886 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:56.904 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:56.904 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:56.905 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:56.905 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:56.905 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:56.905 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:56.905 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:56.905 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:56.906 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:56.924 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:56.925 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:56.926 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:56.926 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:56.926 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:56.926 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:56.926 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:56.926 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:56.927 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:56.964 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:56.964 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:56.964 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:56.964 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:56.965 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:56.965 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:56.965 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:56.965 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:56.966 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:57.024 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:57.024 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:57.024 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:57.024 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:57.025 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:57.025 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:57.025 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:57.025 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:57.025 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:57.026 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=202), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [3ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [21ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [22ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [22ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [23ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [24ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [41ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [42ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [43ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [43ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [61ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [63ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [80ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [101ms] refreshing cache from master, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [102ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [102ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [103ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [103ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [104ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [104ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [140ms] refreshing cache from master, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [142ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [142ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [142ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [143ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [200ms] refreshing cache from master, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [202ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [202ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [202ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [203ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, deferred=Deferred@381469494(state=PENDING, result=null, callback=(continuation of Deferred@45409363 after retry RPC after error@2133176691) -> (continuation of Deferred@1212871127 after retry RPC after error@941567643) -> (continuation of Deferred@497865655 after retry RPC after error@1330661208) -> (continuation of Deferred@1351340844 after retry RPC after error@295785552) -> (continuation of Deferred@1785404600 after retry RPC after error@659356714) -> (continuation of Deferred@701520220 after retry RPC after error@843619298) -> (continuation of Deferred@16043983 after retry RPC after error@1075433047), errback=(continuation of Deferred@45409363 after retry RPC after error@2133176691) -> (continuation of Deferred@1212871127 after retry RPC after error@941567643) -> (continuation of Deferred@497865655 after retry RPC after error@1330661208) -> (continuation of Deferred@1351340844 after retry RPC after error@295785552) -> (continuation of Deferred@1785404600 after retry RPC after error@659356714) -> (continuation of Deferred@701520220 after retry RPC after error@843619298) -> (continuation of Deferred@16043983 after retry RPC after error@1075433047)))
21:04:57.026 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=202), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [3ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [21ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [22ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [22ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [23ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [24ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [41ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [42ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [43ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [43ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [61ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [63ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [80ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [101ms] refreshing cache from master, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [102ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [102ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [103ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [103ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [104ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [104ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [140ms] refreshing cache from master, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [142ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [142ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [142ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [143ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [200ms] refreshing cache from master, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [202ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [202ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [202ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [203ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, deferred=Deferred@381469494(state=PENDING, result=null, callback=(continuation of Deferred@45409363 after retry RPC after error@2133176691) -> (continuation of Deferred@1212871127 after retry RPC after error@941567643) -> (continuation of Deferred@497865655 after retry RPC after error@1330661208) -> (continuation of Deferred@1351340844 after retry RPC after error@295785552) -> (continuation of Deferred@1785404600 after retry RPC after error@659356714) -> (continuation of Deferred@701520220 after retry RPC after error@843619298) -> (continuation of Deferred@16043983 after retry RPC after error@1075433047), errback=(continuation of Deferred@45409363 after retry RPC after error@2133176691) -> (continuation of Deferred@1212871127 after retry RPC after error@941567643) -> (continuation of Deferred@497865655 after retry RPC after error@1330661208) -> (continuation of Deferred@1351340844 after retry RPC after error@295785552) -> (continuation of Deferred@1785404600 after retry RPC after error@659356714) -> (continuation of Deferred@701520220 after retry RPC after error@843619298) -> (continuation of Deferred@16043983 after retry RPC after error@1075433047)))
21:04:57.044 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:57.045 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:57.045 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:57.045 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:57.045 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:57.045 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:57.045 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:57.045 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:57.046 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:57.064 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:57.064 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:57.065 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:57.065 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:57.065 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:57.065 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:57.065 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:57.065 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:57.065 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:57.084 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:57.084 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:57.085 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:57.085 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:57.085 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:57.085 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:57.085 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:57.085 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:57.086 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:57.104 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:57.104 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:57.105 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:57.105 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:57.105 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:57.105 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:57.105 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:57.105 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:57.106 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:57.124 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:57.124 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:57.125 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:57.125 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:57.125 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:57.125 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:57.125 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:57.125 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:57.126 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:57.164 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:57.164 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:57.164 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:57.164 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:57.165 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:57.165 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:57.165 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:57.165 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:57.166 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:57.224 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:57.224 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:57.225 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:57.225 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:57.225 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:57.225 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:57.225 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:57.225 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:57.226 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:57.264 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:57.264 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:57.265 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:57.265 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:57.265 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:57.265 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:57.265 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:57.265 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:57.266 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:57.266 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=221), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [79ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [119ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [181ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [181ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [181ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [219ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [221ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [221ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [222ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [222ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, deferred=Deferred@855172502(state=PENDING, result=null, callback=(continuation of Deferred@2032016359 after retry RPC after error@1591945108) -> (continuation of Deferred@1167198210 after retry RPC after error@808681843) -> (continuation of Deferred@972776881 after retry RPC after error@2120432285) -> (continuation of Deferred@728039243 after retry RPC after error@133221197) -> (continuation of Deferred@1707405551 after retry RPC after error@34900248) -> (continuation of Deferred@235315382 after retry RPC after error@254650520) -> (continuation of Deferred@1303147034 after retry RPC after error@1809215230), errback=(continuation of Deferred@2032016359 after retry RPC after error@1591945108) -> (continuation of Deferred@1167198210 after retry RPC after error@808681843) -> (continuation of Deferred@972776881 after retry RPC after error@2120432285) -> (continuation of Deferred@728039243 after retry RPC after error@133221197) -> (continuation of Deferred@1707405551 after retry RPC after error@34900248) -> (continuation of Deferred@235315382 after retry RPC after error@254650520) -> (continuation of Deferred@1303147034 after retry RPC after error@1809215230)))
21:04:57.267 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=221), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [79ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [119ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [181ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [181ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [181ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [219ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [221ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [221ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [222ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [222ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, deferred=Deferred@855172502(state=PENDING, result=null, callback=(continuation of Deferred@2032016359 after retry RPC after error@1591945108) -> (continuation of Deferred@1167198210 after retry RPC after error@808681843) -> (continuation of Deferred@972776881 after retry RPC after error@2120432285) -> (continuation of Deferred@728039243 after retry RPC after error@133221197) -> (continuation of Deferred@1707405551 after retry RPC after error@34900248) -> (continuation of Deferred@235315382 after retry RPC after error@254650520) -> (continuation of Deferred@1303147034 after retry RPC after error@1809215230), errback=(continuation of Deferred@2032016359 after retry RPC after error@1591945108) -> (continuation of Deferred@1167198210 after retry RPC after error@808681843) -> (continuation of Deferred@972776881 after retry RPC after error@2120432285) -> (continuation of Deferred@728039243 after retry RPC after error@133221197) -> (continuation of Deferred@1707405551 after retry RPC after error@34900248) -> (continuation of Deferred@235315382 after retry RPC after error@254650520) -> (continuation of Deferred@1303147034 after retry RPC after error@1809215230)))
21:04:57.284 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:57.284 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:57.284 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:57.284 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:57.284 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:57.285 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:57.285 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:57.285 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:57.286 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:57.304 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:57.305 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:57.305 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:57.305 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:57.305 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:57.305 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:57.305 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:57.305 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:57.306 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:57.324 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:57.324 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:57.325 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:57.325 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:57.325 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:57.325 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:57.325 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:57.325 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:57.326 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:57.344 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:57.344 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:57.345 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:57.345 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:57.345 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:57.345 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:57.345 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:57.345 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:57.346 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:57.364 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:57.364 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:57.365 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:57.365 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:57.365 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:57.365 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:57.365 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:57.365 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:57.366 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:57.384 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:57.384 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:57.384 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:57.384 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:57.385 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:57.386 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:57.386 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:57.386 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:57.386 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:57.404 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:57.405 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:57.405 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:57.405 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:57.405 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:57.405 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:57.405 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:57.405 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:57.406 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:57.544 [DEBUG - pool-40-thread-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=260), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [21ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [22ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [22ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [23ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [41ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [43ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [61ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [63ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [81ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [103ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [103ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [103ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [121ms] refreshing cache from master, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [123ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, deferred=Deferred@1341776337(state=PENDING, result=null, callback=(continuation of Deferred@728356925 after retry RPC after error@1635215193) -> (continuation of Deferred@1093513506 after retry RPC after error@1096431551) -> (continuation of Deferred@613918617 after retry RPC after error@2122334717) -> (continuation of Deferred@1384663982 after retry RPC after error@1124039055) -> (continuation of Deferred@1540207910 after retry RPC after error@658409483) -> (continuation of Deferred@930837113 after retry RPC after error@250693144) -> (continuation of Deferred@941864549 after retry RPC after error@1796087738), errback=(continuation of Deferred@728356925 after retry RPC after error@1635215193) -> (continuation of Deferred@1093513506 after retry RPC after error@1096431551) -> (continuation of Deferred@613918617 after retry RPC after error@2122334717) -> (continuation of Deferred@1384663982 after retry RPC after error@1124039055) -> (continuation of Deferred@1540207910 after retry RPC after error@658409483) -> (continuation of Deferred@930837113 after retry RPC after error@250693144) -> (continuation of Deferred@941864549 after retry RPC after error@1796087738)))
21:04:57.545 [WARN - pool-40-thread-1] (HashedWheelTimer.java:716) An exception was thrown by TimerTask.
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343) ~[main/:?]
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at java.lang.Thread.run(Thread.java:748) [?:1.8.0_275]
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86)
at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106)
at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198)
at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158)
at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953)
at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923)
at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464)
at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713)
at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701)
at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788)
at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501)
at java.lang.Thread.run(Thread.java:748)
21:04:57.564 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:57.564 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:57.564 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:57.564 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:57.565 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:57.565 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:57.565 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:57.565 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:57.566 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:57.584 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:57.585 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:57.585 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:57.585 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:57.585 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:57.585 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:57.585 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:57.585 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:57.586 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:57.604 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:57.604 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:57.605 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:57.605 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:57.605 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:57.605 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:57.605 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:57.605 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:57.606 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:57.624 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:57.624 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:57.625 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:57.625 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:57.625 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:57.625 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:57.625 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:57.625 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:57.626 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:57.644 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:57.645 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:57.645 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:57.645 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:57.645 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:57.645 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:57.645 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:57.645 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:57.646 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:57.684 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:57.684 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:57.685 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:57.685 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:57.685 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:57.685 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:57.685 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:57.685 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:57.686 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:57.744 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:57.744 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:57.745 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:57.745 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:57.745 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:57.745 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:57.745 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:57.745 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:57.746 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:57.746 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=182), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [21ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [22ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [22ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [23ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [41ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [43ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [60ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [81ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [82ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [82ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [83ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [121ms] refreshing cache from master, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [123ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [181ms] refreshing cache from master, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [182ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [182ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [183ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [183ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, deferred=Deferred@1437922292(state=PENDING, result=null, callback=(continuation of Deferred@894672553 after retry RPC after error@1318647219) -> (continuation of Deferred@855178181 after retry RPC after error@1084327229) -> (continuation of Deferred@1764687190 after retry RPC after error@1611835565) -> (continuation of Deferred@364990905 after retry RPC after error@1493542481) -> (continuation of Deferred@1205810282 after retry RPC after error@847100924) -> (continuation of Deferred@766404755 after retry RPC after error@1500341337), errback=(continuation of Deferred@894672553 after retry RPC after error@1318647219) -> (continuation of Deferred@855178181 after retry RPC after error@1084327229) -> (continuation of Deferred@1764687190 after retry RPC after error@1611835565) -> (continuation of Deferred@364990905 after retry RPC after error@1493542481) -> (continuation of Deferred@1205810282 after retry RPC after error@847100924) -> (continuation of Deferred@766404755 after retry RPC after error@1500341337)))
21:04:57.747 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=182), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [21ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [22ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [22ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [23ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [41ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [43ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [60ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [81ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [82ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [82ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [83ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [121ms] refreshing cache from master, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [123ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [181ms] refreshing cache from master, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [182ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [182ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [183ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [183ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, deferred=Deferred@1437922292(state=PENDING, result=null, callback=(continuation of Deferred@894672553 after retry RPC after error@1318647219) -> (continuation of Deferred@855178181 after retry RPC after error@1084327229) -> (continuation of Deferred@1764687190 after retry RPC after error@1611835565) -> (continuation of Deferred@364990905 after retry RPC after error@1493542481) -> (continuation of Deferred@1205810282 after retry RPC after error@847100924) -> (continuation of Deferred@766404755 after retry RPC after error@1500341337), errback=(continuation of Deferred@894672553 after retry RPC after error@1318647219) -> (continuation of Deferred@855178181 after retry RPC after error@1084327229) -> (continuation of Deferred@1764687190 after retry RPC after error@1611835565) -> (continuation of Deferred@364990905 after retry RPC after error@1493542481) -> (continuation of Deferred@1205810282 after retry RPC after error@847100924) -> (continuation of Deferred@766404755 after retry RPC after error@1500341337)))
21:04:57.764 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:57.764 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:57.765 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:57.765 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:57.765 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:57.765 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:57.765 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:57.765 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:57.766 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:57.785 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:57.785 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:57.785 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:57.785 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:57.785 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:57.785 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:57.786 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:57.786 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:57.786 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:57.804 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:57.804 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:57.805 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:57.805 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:57.805 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:57.805 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:57.805 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:57.806 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:57.806 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:57.824 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:57.824 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:57.825 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:57.825 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:57.825 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:57.825 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:57.825 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:57.825 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:57.826 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:57.844 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:57.844 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:57.845 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:57.845 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:57.845 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:57.845 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:57.845 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:57.845 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:57.846 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:57.884 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:57.884 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:57.885 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:57.885 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:57.885 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:57.885 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:57.885 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:57.885 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:57.886 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:57.964 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:57.964 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:57.965 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:57.965 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:57.965 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:57.965 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:57.965 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:57.965 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:57.966 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:58.024 [DEBUG - pool-40-thread-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=260), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [201ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [201ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [202ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, deferred=Deferred@2061685095(state=PENDING, result=null, callback=(continuation of Deferred@384379802 after retry RPC after error@269443722) -> (continuation of Deferred@1604636445 after retry RPC after error@866210919) -> (continuation of Deferred@1816857714 after retry RPC after error@129710147) -> (continuation of Deferred@2124333966 after retry RPC after error@1193652822) -> (continuation of Deferred@1941507676 after retry RPC after error@756959560) -> (continuation of Deferred@630488534 after retry RPC after error@1330236464) -> (continuation of Deferred@1958338667 after retry RPC after error@724743101), errback=(continuation of Deferred@384379802 after retry RPC after error@269443722) -> (continuation of Deferred@1604636445 after retry RPC after error@866210919) -> (continuation of Deferred@1816857714 after retry RPC after error@129710147) -> (continuation of Deferred@2124333966 after retry RPC after error@1193652822) -> (continuation of Deferred@1941507676 after retry RPC after error@756959560) -> (continuation of Deferred@630488534 after retry RPC after error@1330236464) -> (continuation of Deferred@1958338667 after retry RPC after error@724743101)))
21:04:58.025 [WARN - pool-40-thread-1] (HashedWheelTimer.java:716) An exception was thrown by TimerTask.
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343) ~[main/:?]
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at java.lang.Thread.run(Thread.java:748) [?:1.8.0_275]
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86)
at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106)
at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198)
at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158)
at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953)
at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923)
at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464)
at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713)
at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701)
at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788)
at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501)
at java.lang.Thread.run(Thread.java:748)
21:04:58.044 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:58.044 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:58.045 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:58.045 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:58.045 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:58.045 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:58.045 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:58.045 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:58.046 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:58.064 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:58.064 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:58.064 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:58.064 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:58.064 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:58.065 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:58.065 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:58.065 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:58.066 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:58.084 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:58.085 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:58.085 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:58.085 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:58.085 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:58.085 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:58.085 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:58.086 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:58.086 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:58.104 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:58.104 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:58.105 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:58.105 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:58.105 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:58.105 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:58.105 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:58.105 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:58.106 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:58.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:58.113592 23569 master_proxy_rpc.cc:177] Failed to determine new leader Master: Timed out: ConnectToClusterRpc(addrs: 127.22.168.62:37239,127.22.168.61:32947,127.22.168.60:33247, num_attempts: 97) passed its deadline: Not found: no leader found: ConnectToClusterRpc(addrs: 127.22.168.62:37239,127.22.168.61:32947,127.22.168.60:33247, num_attempts: 1)
21:04:58.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:58.116977 23569 meta_cache.cc:788] Timed out: LookupRpcById { tablet: '01aa0e192c434b468ec7ed90d48563b3', attempt: 2 } failed: LookupRpcById timed out after deadline expired: LookupRpcById { tablet: '01aa0e192c434b468ec7ed90d48563b3', attempt: 2 } passed its deadline: Timed out: ConnectToClusterRpc(addrs: 127.22.168.62:37239,127.22.168.61:32947,127.22.168.60:33247, num_attempts: 97) passed its deadline: Not found: no leader found: ConnectToClusterRpc(addrs: 127.22.168.62:37239,127.22.168.61:32947,127.22.168.60:33247, num_attempts: 1)
21:04:58.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:58.119309 23569 master_proxy_rpc.cc:203] Re-attempting LookupRpcById request to leader Master (127.22.168.60:33247)
21:04:58.124 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:58.124 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:58.125 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:58.125 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:58.125 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:58.126 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:58.126 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:58.126 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:58.126 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:58.144 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:58.144 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:58.145 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:58.145 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:58.145 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:58.145 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:58.145 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:58.146 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:58.146 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:58.224 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:58.225 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:58.225 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:58.225 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:58.225 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:58.225 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:58.225 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:58.225 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:58.226 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:58.227 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=182), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [19ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [82ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [181ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [181ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [182ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, deferred=Deferred@244840830(state=PENDING, result=null, callback=(continuation of Deferred@1433369694 after retry RPC after error@1102030160) -> (continuation of Deferred@1180952468 after retry RPC after error@1997453871) -> (continuation of Deferred@1504556316 after retry RPC after error@1704205916) -> (continuation of Deferred@1016207933 after retry RPC after error@1045368898) -> (continuation of Deferred@176052162 after retry RPC after error@1331551791) -> (continuation of Deferred@1664251403 after retry RPC after error@624535343), errback=(continuation of Deferred@1433369694 after retry RPC after error@1102030160) -> (continuation of Deferred@1180952468 after retry RPC after error@1997453871) -> (continuation of Deferred@1504556316 after retry RPC after error@1704205916) -> (continuation of Deferred@1016207933 after retry RPC after error@1045368898) -> (continuation of Deferred@176052162 after retry RPC after error@1331551791) -> (continuation of Deferred@1664251403 after retry RPC after error@624535343)))
21:04:58.227 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=182), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [19ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [82ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [181ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [181ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [182ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, deferred=Deferred@244840830(state=PENDING, result=null, callback=(continuation of Deferred@1433369694 after retry RPC after error@1102030160) -> (continuation of Deferred@1180952468 after retry RPC after error@1997453871) -> (continuation of Deferred@1504556316 after retry RPC after error@1704205916) -> (continuation of Deferred@1016207933 after retry RPC after error@1045368898) -> (continuation of Deferred@176052162 after retry RPC after error@1331551791) -> (continuation of Deferred@1664251403 after retry RPC after error@624535343), errback=(continuation of Deferred@1433369694 after retry RPC after error@1102030160) -> (continuation of Deferred@1180952468 after retry RPC after error@1997453871) -> (continuation of Deferred@1504556316 after retry RPC after error@1704205916) -> (continuation of Deferred@1016207933 after retry RPC after error@1045368898) -> (continuation of Deferred@176052162 after retry RPC after error@1331551791) -> (continuation of Deferred@1664251403 after retry RPC after error@624535343)))
21:04:58.244 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:58.244 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:58.245 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:58.245 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:58.245 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:58.245 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:58.245 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:58.245 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:58.246 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:58.264 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:58.264 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:58.265 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:58.265 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:58.265 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:58.265 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:58.265 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:58.265 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:58.266 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:58.284 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:58.285 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:58.285 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:58.285 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:58.285 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:58.285 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:58.285 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:58.285 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:58.286 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:58.304 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:58.304 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:58.305 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:58.305 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:58.305 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:58.305 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:58.305 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:58.305 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:58.306 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:58.324 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:58.324 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:58.325 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:58.325 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:58.325 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:58.325 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:58.325 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:58.325 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:58.326 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:58.364 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:58.364 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:58.365 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:58.365 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:58.365 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:58.365 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:58.365 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:58.365 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:58.366 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:58.444 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:58.444 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:58.445 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:58.445 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:58.445 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:58.445 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:58.445 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:58.445 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:58.446 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:58.446 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:58.447 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:58.447 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:58.448 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=203), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [200ms] refreshing cache from master, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [201ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [202ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [203ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [203ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, deferred=Deferred@1990060541(state=PENDING, result=null, callback=(continuation of Deferred@906533241 after retry RPC after error@1843804845) -> (continuation of Deferred@688287146 after retry RPC after error@810396534) -> (continuation of Deferred@1115438935 after retry RPC after error@504579815) -> (continuation of Deferred@81920946 after retry RPC after error@1504251459) -> (continuation of Deferred@2047607974 after retry RPC after error@2116833662) -> (continuation of Deferred@1791439514 after retry RPC after error@2064511639), errback=(continuation of Deferred@906533241 after retry RPC after error@1843804845) -> (continuation of Deferred@688287146 after retry RPC after error@810396534) -> (continuation of Deferred@1115438935 after retry RPC after error@504579815) -> (continuation of Deferred@81920946 after retry RPC after error@1504251459) -> (continuation of Deferred@2047607974 after retry RPC after error@2116833662) -> (continuation of Deferred@1791439514 after retry RPC after error@2064511639)))
21:04:58.448 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=203), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [200ms] refreshing cache from master, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [201ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [202ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [203ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [203ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, deferred=Deferred@1990060541(state=PENDING, result=null, callback=(continuation of Deferred@906533241 after retry RPC after error@1843804845) -> (continuation of Deferred@688287146 after retry RPC after error@810396534) -> (continuation of Deferred@1115438935 after retry RPC after error@504579815) -> (continuation of Deferred@81920946 after retry RPC after error@1504251459) -> (continuation of Deferred@2047607974 after retry RPC after error@2116833662) -> (continuation of Deferred@1791439514 after retry RPC after error@2064511639), errback=(continuation of Deferred@906533241 after retry RPC after error@1843804845) -> (continuation of Deferred@688287146 after retry RPC after error@810396534) -> (continuation of Deferred@1115438935 after retry RPC after error@504579815) -> (continuation of Deferred@81920946 after retry RPC after error@1504251459) -> (continuation of Deferred@2047607974 after retry RPC after error@2116833662) -> (continuation of Deferred@1791439514 after retry RPC after error@2064511639)))
21:04:58.464 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:58.464 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:58.465 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:58.465 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:58.465 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:58.465 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:58.465 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:58.465 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:58.466 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:58.484 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:58.484 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:58.484 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:58.485 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:58.485 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:58.485 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:58.485 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:58.485 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:58.486 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:58.504 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:58.505 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:58.505 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:58.505 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:58.505 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:58.506 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:58.506 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:58.506 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:58.506 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:58.524 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:58.524 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:58.525 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:58.525 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:58.525 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:58.525 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:58.525 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:58.525 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:58.526 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:58.544 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:58.544 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:58.545 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:58.545 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:58.545 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:58.545 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:58.545 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:58.545 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:58.546 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:58.584 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:58.584 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:58.585 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:58.585 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:58.585 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:58.585 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:58.585 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:58.585 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:58.586 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:58.604 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:58.605 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:58.605 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:58.605 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:58.605 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:58.605 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:58.605 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:58.605 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:58.606 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:58.685 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:58.685 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:58.685 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:58.685 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:58.685 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:58.685 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:58.686 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:58.686 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:58.687 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:58.687 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=222), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [119ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [141ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [141ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [142ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [221ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [222ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [222ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [223ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, deferred=Deferred@58489514(state=PENDING, result=null, callback=(continuation of Deferred@930721578 after retry RPC after error@1323094221) -> (continuation of Deferred@1927435773 after retry RPC after error@304863265) -> (continuation of Deferred@1758537137 after retry RPC after error@1084480222) -> (continuation of Deferred@546189962 after retry RPC after error@1814568042) -> (continuation of Deferred@229986547 after retry RPC after error@1394482865) -> (continuation of Deferred@47541306 after retry RPC after error@783481670) -> (continuation of Deferred@161940651 after retry RPC after error@720337080), errback=(continuation of Deferred@930721578 after retry RPC after error@1323094221) -> (continuation of Deferred@1927435773 after retry RPC after error@304863265) -> (continuation of Deferred@1758537137 after retry RPC after error@1084480222) -> (continuation of Deferred@546189962 after retry RPC after error@1814568042) -> (continuation of Deferred@229986547 after retry RPC after error@1394482865) -> (continuation of Deferred@47541306 after retry RPC after error@783481670) -> (continuation of Deferred@161940651 after retry RPC after error@720337080)))
21:04:58.688 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=222), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [119ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [141ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [141ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [142ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [221ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [222ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [222ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [223ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, deferred=Deferred@58489514(state=PENDING, result=null, callback=(continuation of Deferred@930721578 after retry RPC after error@1323094221) -> (continuation of Deferred@1927435773 after retry RPC after error@304863265) -> (continuation of Deferred@1758537137 after retry RPC after error@1084480222) -> (continuation of Deferred@546189962 after retry RPC after error@1814568042) -> (continuation of Deferred@229986547 after retry RPC after error@1394482865) -> (continuation of Deferred@47541306 after retry RPC after error@783481670) -> (continuation of Deferred@161940651 after retry RPC after error@720337080), errback=(continuation of Deferred@930721578 after retry RPC after error@1323094221) -> (continuation of Deferred@1927435773 after retry RPC after error@304863265) -> (continuation of Deferred@1758537137 after retry RPC after error@1084480222) -> (continuation of Deferred@546189962 after retry RPC after error@1814568042) -> (continuation of Deferred@229986547 after retry RPC after error@1394482865) -> (continuation of Deferred@47541306 after retry RPC after error@783481670) -> (continuation of Deferred@161940651 after retry RPC after error@720337080)))
21:04:58.704 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:58.704 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:58.705 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:58.705 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:58.705 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:58.705 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:58.705 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:58.705 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:58.706 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:58.724 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:58.724 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:58.725 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:58.725 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:58.725 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:58.725 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:58.725 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:58.725 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:58.726 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:58.744 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:58.744 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:58.745 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:58.745 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:58.745 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:58.745 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:58.745 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:58.745 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:58.746 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:58.764 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:58.765 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:58.765 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:58.765 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:58.765 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:58.765 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:58.765 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:58.766 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:58.767 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:58.784 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:58.784 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:58.785 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:58.785 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:58.785 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:58.785 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:58.785 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:58.785 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:58.786 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:58.824 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:58.824 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:58.825 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:58.825 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:58.825 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:58.825 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:58.825 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:58.825 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:58.826 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:58.864 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:58.864 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:58.865 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:58.865 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:58.865 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:58.865 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:58.865 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:58.865 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:58.866 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:58.884 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:58.884 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:58.884 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:58.884 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:58.885 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:58.885 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:58.885 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:58.885 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:58.886 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:58.886 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=181), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [39ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [161ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [161ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [161ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [179ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [181ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [181ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [181ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, deferred=Deferred@1241413912(state=PENDING, result=null, callback=(continuation of Deferred@241718008 after retry RPC after error@1431114410) -> (continuation of Deferred@19244602 after retry RPC after error@1000942015) -> (continuation of Deferred@2006602656 after retry RPC after error@1249888187) -> (continuation of Deferred@1715973304 after retry RPC after error@340890910) -> (continuation of Deferred@838062951 after retry RPC after error@84248829) -> (continuation of Deferred@1585239689 after retry RPC after error@1184715069) -> (continuation of Deferred@1767557730 after retry RPC after error@1085400532), errback=(continuation of Deferred@241718008 after retry RPC after error@1431114410) -> (continuation of Deferred@19244602 after retry RPC after error@1000942015) -> (continuation of Deferred@2006602656 after retry RPC after error@1249888187) -> (continuation of Deferred@1715973304 after retry RPC after error@340890910) -> (continuation of Deferred@838062951 after retry RPC after error@84248829) -> (continuation of Deferred@1585239689 after retry RPC after error@1184715069) -> (continuation of Deferred@1767557730 after retry RPC after error@1085400532)))
21:04:58.886 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=181), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [39ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [161ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [161ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [161ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [179ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [181ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [181ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [181ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, deferred=Deferred@1241413912(state=PENDING, result=null, callback=(continuation of Deferred@241718008 after retry RPC after error@1431114410) -> (continuation of Deferred@19244602 after retry RPC after error@1000942015) -> (continuation of Deferred@2006602656 after retry RPC after error@1249888187) -> (continuation of Deferred@1715973304 after retry RPC after error@340890910) -> (continuation of Deferred@838062951 after retry RPC after error@84248829) -> (continuation of Deferred@1585239689 after retry RPC after error@1184715069) -> (continuation of Deferred@1767557730 after retry RPC after error@1085400532), errback=(continuation of Deferred@241718008 after retry RPC after error@1431114410) -> (continuation of Deferred@19244602 after retry RPC after error@1000942015) -> (continuation of Deferred@2006602656 after retry RPC after error@1249888187) -> (continuation of Deferred@1715973304 after retry RPC after error@340890910) -> (continuation of Deferred@838062951 after retry RPC after error@84248829) -> (continuation of Deferred@1585239689 after retry RPC after error@1184715069) -> (continuation of Deferred@1767557730 after retry RPC after error@1085400532)))
21:04:58.904 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:58.905 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:58.905 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:58.905 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:58.905 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:58.905 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:58.905 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:58.905 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:58.906 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:58.924 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:58.924 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:58.925 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:58.925 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:58.925 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:58.925 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:58.925 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:58.925 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:58.926 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:58.944 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:58.944 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:58.945 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:58.945 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:58.945 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:58.945 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:58.945 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:58.945 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:58.946 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:58.964 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:58.964 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:58.964 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:58.964 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:58.965 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:58.965 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:58.965 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:58.965 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:58.965 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:58.984 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:58.985 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:58.985 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:58.985 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:58.985 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:58.985 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:58.985 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:58.985 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:58.986 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:59.024 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:59.024 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:59.025 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:59.025 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:59.025 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:59.025 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:59.025 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:59.025 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:59.026 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:59.044 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:59.044 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:59.045 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:59.045 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:59.045 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:59.045 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:59.045 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:59.045 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:59.046 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:59.064 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:59.064 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:59.064 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:59.064 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:59.064 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:59.065 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:59.065 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:59.065 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:59.065 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:59.144 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:59.144 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:59.144 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:59.144 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:59.145 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:59.145 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:59.145 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:59.145 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:59.145 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:59.146 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=9, TimeoutTracker(timeout=250, elapsed=241), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [59ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [61ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [141ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [141ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [141ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [159ms] refreshing cache from master, [159ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [161ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [161ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [161ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [239ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [241ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [241ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [241ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [242ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, deferred=Deferred@734929653(state=PENDING, result=null, callback=(continuation of Deferred@564149366 after retry RPC after error@150192721) -> (continuation of Deferred@1232193253 after retry RPC after error@334662836) -> (continuation of Deferred@865161520 after retry RPC after error@627502169) -> (continuation of Deferred@1650938110 after retry RPC after error@1710749482) -> (continuation of Deferred@1557537397 after retry RPC after error@1312823698) -> (continuation of Deferred@376983556 after retry RPC after error@1282190150) -> (continuation of Deferred@566558004 after retry RPC after error@1862724631) -> (continuation of Deferred@1001027971 after retry RPC after error@305513259), errback=(continuation of Deferred@564149366 after retry RPC after error@150192721) -> (continuation of Deferred@1232193253 after retry RPC after error@334662836) -> (continuation of Deferred@865161520 after retry RPC after error@627502169) -> (continuation of Deferred@1650938110 after retry RPC after error@1710749482) -> (continuation of Deferred@1557537397 after retry RPC after error@1312823698) -> (continuation of Deferred@376983556 after retry RPC after error@1282190150) -> (continuation of Deferred@566558004 after retry RPC after error@1862724631) -> (continuation of Deferred@1001027971 after retry RPC after error@305513259)))
21:04:59.147 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=9, TimeoutTracker(timeout=250, elapsed=241), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [59ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [61ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [141ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [141ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [141ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [159ms] refreshing cache from master, [159ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [161ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [161ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [161ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [239ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [241ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [241ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [241ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [242ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, deferred=Deferred@734929653(state=PENDING, result=null, callback=(continuation of Deferred@564149366 after retry RPC after error@150192721) -> (continuation of Deferred@1232193253 after retry RPC after error@334662836) -> (continuation of Deferred@865161520 after retry RPC after error@627502169) -> (continuation of Deferred@1650938110 after retry RPC after error@1710749482) -> (continuation of Deferred@1557537397 after retry RPC after error@1312823698) -> (continuation of Deferred@376983556 after retry RPC after error@1282190150) -> (continuation of Deferred@566558004 after retry RPC after error@1862724631) -> (continuation of Deferred@1001027971 after retry RPC after error@305513259), errback=(continuation of Deferred@564149366 after retry RPC after error@150192721) -> (continuation of Deferred@1232193253 after retry RPC after error@334662836) -> (continuation of Deferred@865161520 after retry RPC after error@627502169) -> (continuation of Deferred@1650938110 after retry RPC after error@1710749482) -> (continuation of Deferred@1557537397 after retry RPC after error@1312823698) -> (continuation of Deferred@376983556 after retry RPC after error@1282190150) -> (continuation of Deferred@566558004 after retry RPC after error@1862724631) -> (continuation of Deferred@1001027971 after retry RPC after error@305513259)))
21:04:59.164 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:59.165 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:59.165 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:59.165 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:59.165 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:59.165 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:59.165 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:59.165 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:59.166 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:59.184 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:59.184 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:59.185 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:59.185 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:59.185 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:59.185 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:59.185 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:59.185 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:59.186 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:59.204 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:59.204 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:59.204 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:59.205 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:59.205 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:59.204 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:59.205 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:59.205 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:59.206 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:59.224 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:59.225 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:59.225 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:59.225 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:59.225 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:59.225 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:59.225 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:59.225 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:59.226 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:59.244 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:59.244 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:59.245 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:59.245 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:59.245 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:59.245 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:59.245 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:59.245 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:59.246 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:59.284 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:59.284 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:59.285 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:59.285 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:59.285 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:59.285 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:59.285 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:59.285 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:59.286 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:59.324 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:59.324 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:59.325 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:59.325 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:59.325 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:59.325 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:59.325 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:59.325 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:59.326 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:59.384 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:59.384 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:59.385 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:59.385 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:59.385 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:59.385 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:59.385 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:59.385 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:59.386 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:59.386 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=221), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [39ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [161ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [161ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [162ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [221ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [221ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [222ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [222ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, deferred=Deferred@622705630(state=PENDING, result=null, callback=(continuation of Deferred@1342433789 after retry RPC after error@550223470) -> (continuation of Deferred@1669125272 after retry RPC after error@729239650) -> (continuation of Deferred@1685326064 after retry RPC after error@431088350) -> (continuation of Deferred@485240164 after retry RPC after error@218729179) -> (continuation of Deferred@1233002498 after retry RPC after error@1599048929) -> (continuation of Deferred@573902177 after retry RPC after error@1672156334) -> (continuation of Deferred@1542280937 after retry RPC after error@2028401137), errback=(continuation of Deferred@1342433789 after retry RPC after error@550223470) -> (continuation of Deferred@1669125272 after retry RPC after error@729239650) -> (continuation of Deferred@1685326064 after retry RPC after error@431088350) -> (continuation of Deferred@485240164 after retry RPC after error@218729179) -> (continuation of Deferred@1233002498 after retry RPC after error@1599048929) -> (continuation of Deferred@573902177 after retry RPC after error@1672156334) -> (continuation of Deferred@1542280937 after retry RPC after error@2028401137)))
21:04:59.387 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=221), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [39ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [161ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [161ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [162ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [221ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [221ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [222ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [222ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, deferred=Deferred@622705630(state=PENDING, result=null, callback=(continuation of Deferred@1342433789 after retry RPC after error@550223470) -> (continuation of Deferred@1669125272 after retry RPC after error@729239650) -> (continuation of Deferred@1685326064 after retry RPC after error@431088350) -> (continuation of Deferred@485240164 after retry RPC after error@218729179) -> (continuation of Deferred@1233002498 after retry RPC after error@1599048929) -> (continuation of Deferred@573902177 after retry RPC after error@1672156334) -> (continuation of Deferred@1542280937 after retry RPC after error@2028401137), errback=(continuation of Deferred@1342433789 after retry RPC after error@550223470) -> (continuation of Deferred@1669125272 after retry RPC after error@729239650) -> (continuation of Deferred@1685326064 after retry RPC after error@431088350) -> (continuation of Deferred@485240164 after retry RPC after error@218729179) -> (continuation of Deferred@1233002498 after retry RPC after error@1599048929) -> (continuation of Deferred@573902177 after retry RPC after error@1672156334) -> (continuation of Deferred@1542280937 after retry RPC after error@2028401137)))
21:04:59.404 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:59.404 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:59.404 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:59.404 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:59.405 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:59.405 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:59.405 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:59.405 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:59.406 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:59.424 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:59.425 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:59.425 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:59.425 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:59.425 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:59.425 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:59.425 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:59.425 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:59.426 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:59.444 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:59.444 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:59.445 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:59.445 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:59.445 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:59.445 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:59.445 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:59.445 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:59.446 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:59.464 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:59.464 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:59.465 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:59.465 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:59.465 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:59.465 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:59.465 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:59.465 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:59.466 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:59.484 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:59.484 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:59.484 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:59.484 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:59.485 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:59.485 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:59.485 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:59.485 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:59.485 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:59.524 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:59.525 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:59.525 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:59.525 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:59.525 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:59.525 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:59.525 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:59.525 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:59.526 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:59.564 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:59.564 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:59.565 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:59.565 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:59.565 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:59.565 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:59.565 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:59.565 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:59.566 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:59.584 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:59.584 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:59.585 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:59.585 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:59.585 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:59.585 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:59.585 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:59.585 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:59.586 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:59.586 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=182), Traces: [0ms] refreshing cache from master, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [21ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [22ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [22ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [23ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [41ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [43ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [61ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [63ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [121ms] refreshing cache from master, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [122ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [122ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [123ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [161ms] refreshing cache from master, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [162ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [162ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [163ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [163ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [181ms] refreshing cache from master, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [182ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [182ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [182ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [183ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, deferred=Deferred@640641963(state=PENDING, result=null, callback=(continuation of Deferred@919959809 after retry RPC after error@1044425606) -> (continuation of Deferred@1886366010 after retry RPC after error@759228079) -> (continuation of Deferred@817926007 after retry RPC after error@191307378) -> (continuation of Deferred@105660509 after retry RPC after error@739486629) -> (continuation of Deferred@404496999 after retry RPC after error@733686680) -> (continuation of Deferred@1334958106 after retry RPC after error@2121141454) -> (continuation of Deferred@119019134 after retry RPC after error@1972181742), errback=(continuation of Deferred@919959809 after retry RPC after error@1044425606) -> (continuation of Deferred@1886366010 after retry RPC after error@759228079) -> (continuation of Deferred@817926007 after retry RPC after error@191307378) -> (continuation of Deferred@105660509 after retry RPC after error@739486629) -> (continuation of Deferred@404496999 after retry RPC after error@733686680) -> (continuation of Deferred@1334958106 after retry RPC after error@2121141454) -> (continuation of Deferred@119019134 after retry RPC after error@1972181742)))
21:04:59.587 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=182), Traces: [0ms] refreshing cache from master, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [21ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [22ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [22ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [23ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [41ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [43ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [61ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [63ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [121ms] refreshing cache from master, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [122ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [122ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [123ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [161ms] refreshing cache from master, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [162ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [162ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [163ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [163ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [181ms] refreshing cache from master, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [182ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [182ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [182ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [183ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, deferred=Deferred@640641963(state=PENDING, result=null, callback=(continuation of Deferred@919959809 after retry RPC after error@1044425606) -> (continuation of Deferred@1886366010 after retry RPC after error@759228079) -> (continuation of Deferred@817926007 after retry RPC after error@191307378) -> (continuation of Deferred@105660509 after retry RPC after error@739486629) -> (continuation of Deferred@404496999 after retry RPC after error@733686680) -> (continuation of Deferred@1334958106 after retry RPC after error@2121141454) -> (continuation of Deferred@119019134 after retry RPC after error@1972181742), errback=(continuation of Deferred@919959809 after retry RPC after error@1044425606) -> (continuation of Deferred@1886366010 after retry RPC after error@759228079) -> (continuation of Deferred@817926007 after retry RPC after error@191307378) -> (continuation of Deferred@105660509 after retry RPC after error@739486629) -> (continuation of Deferred@404496999 after retry RPC after error@733686680) -> (continuation of Deferred@1334958106 after retry RPC after error@2121141454) -> (continuation of Deferred@119019134 after retry RPC after error@1972181742)))
21:04:59.604 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:59.605 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:59.605 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:59.605 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:59.606 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:59.606 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:59.606 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:59.606 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:59.606 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:59.624 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:59.624 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:59.625 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:59.625 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:59.625 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:59.625 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:59.625 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:59.625 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:59.626 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:59.644 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:59.644 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:59.645 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:59.645 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:59.645 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:59.645 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:59.645 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:59.645 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:59.646 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:59.664 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:59.664 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:59.664 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:59.664 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:59.664 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:59.665 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:59.665 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:59.665 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:59.666 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:59.684 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:59.685 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:59.685 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:59.685 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:59.685 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:59.685 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:59.685 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:59.685 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:59.686 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:59.724 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:59.724 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:59.725 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:59.725 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:59.725 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:59.725 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:59.725 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:59.725 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:59.726 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:59.744 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:59.744 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:59.745 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:59.745 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:59.745 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:59.745 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:59.745 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:59.745 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:59.746 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:59.804 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:59.804 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:59.805 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:59.805 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:59.805 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:59.805 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:59.805 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:59.805 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:59.806 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:59.806 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=201), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [59ms] refreshing cache from master, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [141ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [141ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [142ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [201ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [201ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [202ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, deferred=Deferred@1584445264(state=PENDING, result=null, callback=(continuation of Deferred@493768841 after retry RPC after error@2141949241) -> (continuation of Deferred@1931334400 after retry RPC after error@680275972) -> (continuation of Deferred@1200215940 after retry RPC after error@1684559881) -> (continuation of Deferred@2045772375 after retry RPC after error@808784253) -> (continuation of Deferred@956941576 after retry RPC after error@528968240) -> (continuation of Deferred@1721508141 after retry RPC after error@2117757562) -> (continuation of Deferred@166510896 after retry RPC after error@765390039), errback=(continuation of Deferred@493768841 after retry RPC after error@2141949241) -> (continuation of Deferred@1931334400 after retry RPC after error@680275972) -> (continuation of Deferred@1200215940 after retry RPC after error@1684559881) -> (continuation of Deferred@2045772375 after retry RPC after error@808784253) -> (continuation of Deferred@956941576 after retry RPC after error@528968240) -> (continuation of Deferred@1721508141 after retry RPC after error@2117757562) -> (continuation of Deferred@166510896 after retry RPC after error@765390039)))
21:04:59.807 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=201), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [59ms] refreshing cache from master, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [141ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [141ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [142ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [201ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [201ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [202ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, deferred=Deferred@1584445264(state=PENDING, result=null, callback=(continuation of Deferred@493768841 after retry RPC after error@2141949241) -> (continuation of Deferred@1931334400 after retry RPC after error@680275972) -> (continuation of Deferred@1200215940 after retry RPC after error@1684559881) -> (continuation of Deferred@2045772375 after retry RPC after error@808784253) -> (continuation of Deferred@956941576 after retry RPC after error@528968240) -> (continuation of Deferred@1721508141 after retry RPC after error@2117757562) -> (continuation of Deferred@166510896 after retry RPC after error@765390039), errback=(continuation of Deferred@493768841 after retry RPC after error@2141949241) -> (continuation of Deferred@1931334400 after retry RPC after error@680275972) -> (continuation of Deferred@1200215940 after retry RPC after error@1684559881) -> (continuation of Deferred@2045772375 after retry RPC after error@808784253) -> (continuation of Deferred@956941576 after retry RPC after error@528968240) -> (continuation of Deferred@1721508141 after retry RPC after error@2117757562) -> (continuation of Deferred@166510896 after retry RPC after error@765390039)))
21:04:59.824 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:59.824 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:59.825 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:59.825 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:59.825 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:59.825 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:59.825 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:59.825 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:59.826 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:59.844 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:59.844 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:59.844 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:59.844 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:59.845 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:59.845 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:59.845 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:59.845 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:59.846 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:59.864 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:59.865 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:59.866 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:59.866 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:59.866 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:59.866 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:59.866 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:59.866 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:59.867 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:59.884 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:59.885 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:59.885 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:59.885 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:59.885 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:59.885 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:59.885 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:59.886 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:59.887 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:59.904 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:59.904 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:59.905 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:59.905 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:59.905 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:59.905 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:59.905 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:59.905 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:59.906 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:59.924 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:59.924 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:59.925 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:59.925 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:59.925 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:59.925 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:59.925 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:59.925 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:59.926 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:59.984 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:04:59.984 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:04:59.985 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:59.985 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:59.985 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:59.985 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:59.985 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:04:59.985 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:04:59.986 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:04:59.987 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=162), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [19ms] refreshing cache from master, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [43ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [161ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [161ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [162ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, deferred=Deferred@1102398678(state=PENDING, result=null, callback=(continuation of Deferred@934992130 after retry RPC after error@1241662572) -> (continuation of Deferred@800739197 after retry RPC after error@1153576360) -> (continuation of Deferred@626102011 after retry RPC after error@2108198681) -> (continuation of Deferred@9762422 after retry RPC after error@1441674401) -> (continuation of Deferred@2086756017 after retry RPC after error@1158202979) -> (continuation of Deferred@1653129202 after retry RPC after error@1181205566), errback=(continuation of Deferred@934992130 after retry RPC after error@1241662572) -> (continuation of Deferred@800739197 after retry RPC after error@1153576360) -> (continuation of Deferred@626102011 after retry RPC after error@2108198681) -> (continuation of Deferred@9762422 after retry RPC after error@1441674401) -> (continuation of Deferred@2086756017 after retry RPC after error@1158202979) -> (continuation of Deferred@1653129202 after retry RPC after error@1181205566)))
21:04:59.987 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=162), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [19ms] refreshing cache from master, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [43ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [161ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [161ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [162ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, deferred=Deferred@1102398678(state=PENDING, result=null, callback=(continuation of Deferred@934992130 after retry RPC after error@1241662572) -> (continuation of Deferred@800739197 after retry RPC after error@1153576360) -> (continuation of Deferred@626102011 after retry RPC after error@2108198681) -> (continuation of Deferred@9762422 after retry RPC after error@1441674401) -> (continuation of Deferred@2086756017 after retry RPC after error@1158202979) -> (continuation of Deferred@1653129202 after retry RPC after error@1181205566), errback=(continuation of Deferred@934992130 after retry RPC after error@1241662572) -> (continuation of Deferred@800739197 after retry RPC after error@1153576360) -> (continuation of Deferred@626102011 after retry RPC after error@2108198681) -> (continuation of Deferred@9762422 after retry RPC after error@1441674401) -> (continuation of Deferred@2086756017 after retry RPC after error@1158202979) -> (continuation of Deferred@1653129202 after retry RPC after error@1181205566)))
21:05:00.004 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:05:00.004 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:05:00.005 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:00.005 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:00.005 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:00.005 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:00.005 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:00.005 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:00.006 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:05:00.024 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:05:00.024 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:05:00.025 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:00.025 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:00.025 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:00.025 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:00.025 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:00.025 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:00.026 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:05:00.044 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:05:00.044 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:05:00.044 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:00.044 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:00.045 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:00.045 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:00.045 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:00.045 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:00.046 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:05:00.064 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:05:00.065 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:05:00.065 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:00.065 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:00.065 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:00.065 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:00.065 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:00.065 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:00.066 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:05:00.084 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:05:00.084 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:05:00.085 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:00.085 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:00.085 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:00.085 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:00.085 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:00.085 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:00.086 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:05:00.104 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:05:00.104 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:05:00.105 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:00.105 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:00.105 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:00.105 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:00.105 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:00.105 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:00.106 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:05:00.164 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:05:00.164 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:05:00.165 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:00.165 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:00.165 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:00.165 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:00.165 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:00.165 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:00.165 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:00.165 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:00.166 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:05:00.166 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:05:00.244 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:05:00.244 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:05:00.245 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:00.245 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:00.245 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:00.245 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:00.245 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:00.245 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:00.246 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:05:00.246 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=242), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [39ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [161ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [161ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [162ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [241ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [241ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [242ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [242ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, deferred=Deferred@2005628553(state=PENDING, result=null, callback=(continuation of Deferred@373939696 after retry RPC after error@159212018) -> (continuation of Deferred@1520977058 after retry RPC after error@1983602240) -> (continuation of Deferred@784706346 after retry RPC after error@1289039345) -> (continuation of Deferred@1421933981 after retry RPC after error@2024108059) -> (continuation of Deferred@2067405022 after retry RPC after error@509937901) -> (continuation of Deferred@1301947049 after retry RPC after error@518825641) -> (continuation of Deferred@901493058 after retry RPC after error@53470584), errback=(continuation of Deferred@373939696 after retry RPC after error@159212018) -> (continuation of Deferred@1520977058 after retry RPC after error@1983602240) -> (continuation of Deferred@784706346 after retry RPC after error@1289039345) -> (continuation of Deferred@1421933981 after retry RPC after error@2024108059) -> (continuation of Deferred@2067405022 after retry RPC after error@509937901) -> (continuation of Deferred@1301947049 after retry RPC after error@518825641) -> (continuation of Deferred@901493058 after retry RPC after error@53470584)))
21:05:00.247 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=242), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [39ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [161ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [161ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [162ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [241ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [241ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [242ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [242ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, deferred=Deferred@2005628553(state=PENDING, result=null, callback=(continuation of Deferred@373939696 after retry RPC after error@159212018) -> (continuation of Deferred@1520977058 after retry RPC after error@1983602240) -> (continuation of Deferred@784706346 after retry RPC after error@1289039345) -> (continuation of Deferred@1421933981 after retry RPC after error@2024108059) -> (continuation of Deferred@2067405022 after retry RPC after error@509937901) -> (continuation of Deferred@1301947049 after retry RPC after error@518825641) -> (continuation of Deferred@901493058 after retry RPC after error@53470584), errback=(continuation of Deferred@373939696 after retry RPC after error@159212018) -> (continuation of Deferred@1520977058 after retry RPC after error@1983602240) -> (continuation of Deferred@784706346 after retry RPC after error@1289039345) -> (continuation of Deferred@1421933981 after retry RPC after error@2024108059) -> (continuation of Deferred@2067405022 after retry RPC after error@509937901) -> (continuation of Deferred@1301947049 after retry RPC after error@518825641) -> (continuation of Deferred@901493058 after retry RPC after error@53470584)))
21:05:00.264 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:05:00.264 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:05:00.265 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:00.265 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:00.265 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:00.265 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:00.265 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:00.265 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:00.266 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:05:00.285 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:05:00.285 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:05:00.285 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:00.285 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:00.286 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:00.286 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:00.286 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:00.286 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:00.286 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:05:00.304 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:05:00.304 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:05:00.305 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:00.305 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:00.305 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:00.305 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:00.305 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:00.305 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:00.306 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:05:00.324 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:05:00.324 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:05:00.325 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:00.325 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:00.325 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:00.325 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:00.325 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:00.325 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:00.326 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:05:00.344 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:05:00.344 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:05:00.345 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:00.345 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:00.345 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:00.345 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:00.345 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:00.345 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:00.346 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:05:00.364 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:05:00.364 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:05:00.364 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:00.364 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:00.365 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:00.365 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:00.365 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:00.365 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:00.366 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:05:00.404 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:05:00.405 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:05:00.405 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:00.405 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:00.405 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:00.406 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:00.406 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:00.406 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:00.407 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:05:00.444 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:05:00.445 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:05:00.445 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:00.445 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:00.445 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:00.445 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:00.445 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:00.445 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:00.446 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:05:00.447 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=182), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [79ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [99ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [141ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [142ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [142ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [143ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [181ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [181ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [182ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, deferred=Deferred@940771336(state=PENDING, result=null, callback=(continuation of Deferred@1941556777 after retry RPC after error@1419180272) -> (continuation of Deferred@232032417 after retry RPC after error@765952603) -> (continuation of Deferred@273865024 after retry RPC after error@1935729346) -> (continuation of Deferred@1524166031 after retry RPC after error@1274812885) -> (continuation of Deferred@1298288947 after retry RPC after error@971037871) -> (continuation of Deferred@608823945 after retry RPC after error@265321828) -> (continuation of Deferred@1348755044 after retry RPC after error@1876882787), errback=(continuation of Deferred@1941556777 after retry RPC after error@1419180272) -> (continuation of Deferred@232032417 after retry RPC after error@765952603) -> (continuation of Deferred@273865024 after retry RPC after error@1935729346) -> (continuation of Deferred@1524166031 after retry RPC after error@1274812885) -> (continuation of Deferred@1298288947 after retry RPC after error@971037871) -> (continuation of Deferred@608823945 after retry RPC after error@265321828) -> (continuation of Deferred@1348755044 after retry RPC after error@1876882787)))
21:05:00.447 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=182), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [79ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [99ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [141ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [142ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [142ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [143ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [181ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [181ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [182ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, deferred=Deferred@940771336(state=PENDING, result=null, callback=(continuation of Deferred@1941556777 after retry RPC after error@1419180272) -> (continuation of Deferred@232032417 after retry RPC after error@765952603) -> (continuation of Deferred@273865024 after retry RPC after error@1935729346) -> (continuation of Deferred@1524166031 after retry RPC after error@1274812885) -> (continuation of Deferred@1298288947 after retry RPC after error@971037871) -> (continuation of Deferred@608823945 after retry RPC after error@265321828) -> (continuation of Deferred@1348755044 after retry RPC after error@1876882787), errback=(continuation of Deferred@1941556777 after retry RPC after error@1419180272) -> (continuation of Deferred@232032417 after retry RPC after error@765952603) -> (continuation of Deferred@273865024 after retry RPC after error@1935729346) -> (continuation of Deferred@1524166031 after retry RPC after error@1274812885) -> (continuation of Deferred@1298288947 after retry RPC after error@971037871) -> (continuation of Deferred@608823945 after retry RPC after error@265321828) -> (continuation of Deferred@1348755044 after retry RPC after error@1876882787)))
21:05:00.464 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:05:00.464 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:05:00.465 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:00.465 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:00.465 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:00.465 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:00.465 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:00.465 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:00.466 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:05:00.484 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:05:00.484 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:05:00.485 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:00.485 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:00.485 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:00.485 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:00.485 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:00.485 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:00.486 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:05:00.504 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:05:00.504 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:05:00.505 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:00.505 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:00.505 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:00.505 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:00.505 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:00.505 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:00.506 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:05:00.524 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:05:00.525 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:05:00.525 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:00.525 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:00.525 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:00.525 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:00.526 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:00.526 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:00.526 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:05:00.544 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:05:00.545 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:05:00.545 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:00.545 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:00.545 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:00.545 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:00.546 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:00.546 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:00.547 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:05:00.564 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:05:00.564 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:05:00.565 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:00.565 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:00.565 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:00.566 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:00.566 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:00.566 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:00.566 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:05:00.644 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:05:00.644 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:05:00.644 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:00.644 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:00.645 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:00.645 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:00.645 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:00.645 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:00.645 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:05:00.646 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=181), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [179ms] refreshing cache from master, [179ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [181ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [181ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [181ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, deferred=Deferred@175820343(state=PENDING, result=null, callback=(continuation of Deferred@586220432 after retry RPC after error@148203981) -> (continuation of Deferred@340862757 after retry RPC after error@1208375534) -> (continuation of Deferred@1648443389 after retry RPC after error@1098496377) -> (continuation of Deferred@1552733820 after retry RPC after error@641147947) -> (continuation of Deferred@1298291143 after retry RPC after error@1096586083) -> (continuation of Deferred@2077852162 after retry RPC after error@1691412270), errback=(continuation of Deferred@586220432 after retry RPC after error@148203981) -> (continuation of Deferred@340862757 after retry RPC after error@1208375534) -> (continuation of Deferred@1648443389 after retry RPC after error@1098496377) -> (continuation of Deferred@1552733820 after retry RPC after error@641147947) -> (continuation of Deferred@1298291143 after retry RPC after error@1096586083) -> (continuation of Deferred@2077852162 after retry RPC after error@1691412270)))
21:05:00.646 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=181), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [179ms] refreshing cache from master, [179ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [181ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [181ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [181ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, deferred=Deferred@175820343(state=PENDING, result=null, callback=(continuation of Deferred@586220432 after retry RPC after error@148203981) -> (continuation of Deferred@340862757 after retry RPC after error@1208375534) -> (continuation of Deferred@1648443389 after retry RPC after error@1098496377) -> (continuation of Deferred@1552733820 after retry RPC after error@641147947) -> (continuation of Deferred@1298291143 after retry RPC after error@1096586083) -> (continuation of Deferred@2077852162 after retry RPC after error@1691412270), errback=(continuation of Deferred@586220432 after retry RPC after error@148203981) -> (continuation of Deferred@340862757 after retry RPC after error@1208375534) -> (continuation of Deferred@1648443389 after retry RPC after error@1098496377) -> (continuation of Deferred@1552733820 after retry RPC after error@641147947) -> (continuation of Deferred@1298291143 after retry RPC after error@1096586083) -> (continuation of Deferred@2077852162 after retry RPC after error@1691412270)))
21:05:00.664 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:05:00.665 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:05:00.665 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:00.665 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:00.665 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:00.665 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:00.665 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:00.665 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:00.667 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:05:00.684 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:05:00.684 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:05:00.685 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:00.685 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:00.685 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:00.685 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:00.685 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:00.685 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:00.686 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:05:00.704 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:05:00.704 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:05:00.705 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:00.705 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:00.705 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:00.705 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:00.705 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:00.705 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:00.706 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:05:00.724 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:05:00.724 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:05:00.724 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:00.724 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:00.725 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:00.725 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:00.725 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:00.725 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:00.726 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:05:00.744 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:05:00.745 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:05:00.745 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:00.745 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:00.745 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:00.745 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:00.746 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:00.746 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:00.747 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:05:00.784 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:05:00.785 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:05:00.785 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:00.785 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:00.786 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:00.785 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:00.787 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:00.787 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:00.787 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:05:00.864 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:05:00.864 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:05:00.864 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:00.864 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:00.865 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:00.865 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:00.865 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:00.865 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:00.866 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:05:00.904 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:05:00.904 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:05:00.904 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:00.904 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:00.905 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:00.905 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:00.905 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:00.905 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:00.906 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:05:00.906 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=241), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [59ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [123ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [123ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [199ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [201ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [201ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [202ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [239ms] refreshing cache from master, [239ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [241ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [241ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [242ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [242ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, deferred=Deferred@1601346477(state=PENDING, result=null, callback=(continuation of Deferred@1443218721 after retry RPC after error@1580054034) -> (continuation of Deferred@427042053 after retry RPC after error@1376811637) -> (continuation of Deferred@717521371 after retry RPC after error@434782135) -> (continuation of Deferred@554597762 after retry RPC after error@1889298436) -> (continuation of Deferred@2119759253 after retry RPC after error@914844270) -> (continuation of Deferred@1681388369 after retry RPC after error@2004384667) -> (continuation of Deferred@1312152764 after retry RPC after error@1131154190), errback=(continuation of Deferred@1443218721 after retry RPC after error@1580054034) -> (continuation of Deferred@427042053 after retry RPC after error@1376811637) -> (continuation of Deferred@717521371 after retry RPC after error@434782135) -> (continuation of Deferred@554597762 after retry RPC after error@1889298436) -> (continuation of Deferred@2119759253 after retry RPC after error@914844270) -> (continuation of Deferred@1681388369 after retry RPC after error@2004384667) -> (continuation of Deferred@1312152764 after retry RPC after error@1131154190)))
21:05:00.907 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=241), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [59ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [123ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [123ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [199ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [201ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [201ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [202ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [239ms] refreshing cache from master, [239ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [241ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [241ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [242ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [242ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, deferred=Deferred@1601346477(state=PENDING, result=null, callback=(continuation of Deferred@1443218721 after retry RPC after error@1580054034) -> (continuation of Deferred@427042053 after retry RPC after error@1376811637) -> (continuation of Deferred@717521371 after retry RPC after error@434782135) -> (continuation of Deferred@554597762 after retry RPC after error@1889298436) -> (continuation of Deferred@2119759253 after retry RPC after error@914844270) -> (continuation of Deferred@1681388369 after retry RPC after error@2004384667) -> (continuation of Deferred@1312152764 after retry RPC after error@1131154190), errback=(continuation of Deferred@1443218721 after retry RPC after error@1580054034) -> (continuation of Deferred@427042053 after retry RPC after error@1376811637) -> (continuation of Deferred@717521371 after retry RPC after error@434782135) -> (continuation of Deferred@554597762 after retry RPC after error@1889298436) -> (continuation of Deferred@2119759253 after retry RPC after error@914844270) -> (continuation of Deferred@1681388369 after retry RPC after error@2004384667) -> (continuation of Deferred@1312152764 after retry RPC after error@1131154190)))
21:05:00.924 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:05:00.925 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:05:00.925 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:00.925 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:00.925 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:00.925 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:00.925 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:00.926 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:00.926 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:05:00.944 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:05:00.944 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:05:00.945 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:00.945 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:00.945 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:00.945 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:00.945 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:00.945 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:00.946 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:05:00.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:05:00.952595 24071 master_runner.cc:186] Error getting master registration for 127.22.168.62:37239: Network error: Client connection negotiation failed: client connection to 127.22.168.62:37239: connect: Connection refused (error 111),
21:05:00.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:05:00.952795 24071 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
21:05:00.964 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:05:00.964 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:05:00.965 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:00.965 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:00.965 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:00.965 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:00.965 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:00.965 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:00.966 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:05:00.984 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:05:00.985 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:05:00.985 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:00.985 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:00.985 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:00.985 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:00.986 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:00.986 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:00.986 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:05:01.004 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:05:01.005 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:05:01.005 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:01.005 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:01.005 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:01.006 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:01.006 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:01.006 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:01.006 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:05:01.024 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:05:01.024 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:05:01.025 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:01.025 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:01.025 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:01.025 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:01.025 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:01.025 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:01.026 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:05:01.084 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:05:01.085 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:05:01.085 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:01.085 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:01.085 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:01.085 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:01.085 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:01.085 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:01.086 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:05:01.124 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:05:01.125 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:05:01.125 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:01.125 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:01.125 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:01.125 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:01.125 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:01.125 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:01.126 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:05:01.127 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=202), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [161ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [161ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [162ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [201ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [201ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [202ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, deferred=Deferred@583855837(state=PENDING, result=null, callback=(continuation of Deferred@1794396643 after retry RPC after error@1012846425) -> (continuation of Deferred@2008579669 after retry RPC after error@851958905) -> (continuation of Deferred@2054482059 after retry RPC after error@815453339) -> (continuation of Deferred@82071051 after retry RPC after error@147619578) -> (continuation of Deferred@1436680152 after retry RPC after error@863619067) -> (continuation of Deferred@365646729 after retry RPC after error@2124141668) -> (continuation of Deferred@614378320 after retry RPC after error@1504044583), errback=(continuation of Deferred@1794396643 after retry RPC after error@1012846425) -> (continuation of Deferred@2008579669 after retry RPC after error@851958905) -> (continuation of Deferred@2054482059 after retry RPC after error@815453339) -> (continuation of Deferred@82071051 after retry RPC after error@147619578) -> (continuation of Deferred@1436680152 after retry RPC after error@863619067) -> (continuation of Deferred@365646729 after retry RPC after error@2124141668) -> (continuation of Deferred@614378320 after retry RPC after error@1504044583)))
21:05:01.127 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=202), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [161ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [161ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [162ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [201ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [201ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [202ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, deferred=Deferred@583855837(state=PENDING, result=null, callback=(continuation of Deferred@1794396643 after retry RPC after error@1012846425) -> (continuation of Deferred@2008579669 after retry RPC after error@851958905) -> (continuation of Deferred@2054482059 after retry RPC after error@815453339) -> (continuation of Deferred@82071051 after retry RPC after error@147619578) -> (continuation of Deferred@1436680152 after retry RPC after error@863619067) -> (continuation of Deferred@365646729 after retry RPC after error@2124141668) -> (continuation of Deferred@614378320 after retry RPC after error@1504044583), errback=(continuation of Deferred@1794396643 after retry RPC after error@1012846425) -> (continuation of Deferred@2008579669 after retry RPC after error@851958905) -> (continuation of Deferred@2054482059 after retry RPC after error@815453339) -> (continuation of Deferred@82071051 after retry RPC after error@147619578) -> (continuation of Deferred@1436680152 after retry RPC after error@863619067) -> (continuation of Deferred@365646729 after retry RPC after error@2124141668) -> (continuation of Deferred@614378320 after retry RPC after error@1504044583)))
21:05:01.144 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:05:01.144 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:05:01.145 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:01.145 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:01.145 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:01.145 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:01.145 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:01.145 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:01.146 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:05:01.164 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:05:01.164 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:05:01.165 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:01.165 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:01.165 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:01.165 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:01.165 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:01.165 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:01.166 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:05:01.184 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:05:01.184 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:05:01.185 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:01.185 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:01.185 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:01.185 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:01.185 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:01.185 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:01.186 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:05:01.205 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:05:01.205 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:05:01.205 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:01.205 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:01.205 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:01.206 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:01.206 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:01.206 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:01.207 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:05:01.224 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:05:01.225 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:05:01.225 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:01.225 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:01.225 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:01.225 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:01.226 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:01.226 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:01.226 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:05:01.244 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:05:01.244 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:05:01.245 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:01.245 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:01.245 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:01.245 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:01.245 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:01.245 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:01.246 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:05:01.304 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:05:01.304 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:05:01.305 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:01.305 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:01.305 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:01.305 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:01.305 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:01.305 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:01.306 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:05:01.306 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=162), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [161ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [161ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [162ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, deferred=Deferred@1658352806(state=PENDING, result=null, callback=(continuation of Deferred@1117748546 after retry RPC after error@1631834754) -> (continuation of Deferred@227796709 after retry RPC after error@751816373) -> (continuation of Deferred@352568067 after retry RPC after error@1781257443) -> (continuation of Deferred@2023596941 after retry RPC after error@720724045) -> (continuation of Deferred@568698315 after retry RPC after error@366862808) -> (continuation of Deferred@4831525 after retry RPC after error@20928105), errback=(continuation of Deferred@1117748546 after retry RPC after error@1631834754) -> (continuation of Deferred@227796709 after retry RPC after error@751816373) -> (continuation of Deferred@352568067 after retry RPC after error@1781257443) -> (continuation of Deferred@2023596941 after retry RPC after error@720724045) -> (continuation of Deferred@568698315 after retry RPC after error@366862808) -> (continuation of Deferred@4831525 after retry RPC after error@20928105)))
21:05:01.307 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=162), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [161ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [161ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [162ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, deferred=Deferred@1658352806(state=PENDING, result=null, callback=(continuation of Deferred@1117748546 after retry RPC after error@1631834754) -> (continuation of Deferred@227796709 after retry RPC after error@751816373) -> (continuation of Deferred@352568067 after retry RPC after error@1781257443) -> (continuation of Deferred@2023596941 after retry RPC after error@720724045) -> (continuation of Deferred@568698315 after retry RPC after error@366862808) -> (continuation of Deferred@4831525 after retry RPC after error@20928105), errback=(continuation of Deferred@1117748546 after retry RPC after error@1631834754) -> (continuation of Deferred@227796709 after retry RPC after error@751816373) -> (continuation of Deferred@352568067 after retry RPC after error@1781257443) -> (continuation of Deferred@2023596941 after retry RPC after error@720724045) -> (continuation of Deferred@568698315 after retry RPC after error@366862808) -> (continuation of Deferred@4831525 after retry RPC after error@20928105)))
21:05:01.324 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:05:01.324 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:05:01.325 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:01.325 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:01.325 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:01.325 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:01.325 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:01.325 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:01.326 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:05:01.344 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:05:01.344 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:05:01.345 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:01.345 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:01.345 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:01.345 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:01.345 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:01.345 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:01.346 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:05:01.364 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:05:01.364 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:05:01.364 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:01.365 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:01.365 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:01.364 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:01.365 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:01.365 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:01.366 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:05:01.384 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:05:01.385 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:05:01.385 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:01.385 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:01.385 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:01.385 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:01.385 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:01.385 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:01.386 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:05:01.404 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:05:01.404 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:05:01.405 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:01.405 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:01.405 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:01.405 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:01.405 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:01.405 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:01.406 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:05:01.424 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:05:01.424 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:05:01.425 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:01.425 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:01.425 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:01.425 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:01.425 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:01.425 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:01.426 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:05:01.444 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:05:01.444 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:05:01.444 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:01.444 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:01.445 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:01.445 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:01.445 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:01.445 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:01.446 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:05:01.464 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:05:01.465 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:05:01.465 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:01.465 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:01.465 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:01.465 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:01.466 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:01.466 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:01.467 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:05:01.467 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=142), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [39ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [119ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [141ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [142ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [142ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [143ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, deferred=Deferred@138001660(state=PENDING, result=null, callback=(continuation of Deferred@142807095 after retry RPC after error@1525045814) -> (continuation of Deferred@1112205807 after retry RPC after error@131166982) -> (continuation of Deferred@390318435 after retry RPC after error@2140935767) -> (continuation of Deferred@1779211738 after retry RPC after error@1716099806) -> (continuation of Deferred@1867009957 after retry RPC after error@1887027893) -> (continuation of Deferred@2123719469 after retry RPC after error@1411422539) -> (continuation of Deferred@2033762573 after retry RPC after error@1206569205), errback=(continuation of Deferred@142807095 after retry RPC after error@1525045814) -> (continuation of Deferred@1112205807 after retry RPC after error@131166982) -> (continuation of Deferred@390318435 after retry RPC after error@2140935767) -> (continuation of Deferred@1779211738 after retry RPC after error@1716099806) -> (continuation of Deferred@1867009957 after retry RPC after error@1887027893) -> (continuation of Deferred@2123719469 after retry RPC after error@1411422539) -> (continuation of Deferred@2033762573 after retry RPC after error@1206569205)))
21:05:01.467 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=142), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [39ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [119ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [141ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [142ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [142ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [143ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, deferred=Deferred@138001660(state=PENDING, result=null, callback=(continuation of Deferred@142807095 after retry RPC after error@1525045814) -> (continuation of Deferred@1112205807 after retry RPC after error@131166982) -> (continuation of Deferred@390318435 after retry RPC after error@2140935767) -> (continuation of Deferred@1779211738 after retry RPC after error@1716099806) -> (continuation of Deferred@1867009957 after retry RPC after error@1887027893) -> (continuation of Deferred@2123719469 after retry RPC after error@1411422539) -> (continuation of Deferred@2033762573 after retry RPC after error@1206569205), errback=(continuation of Deferred@142807095 after retry RPC after error@1525045814) -> (continuation of Deferred@1112205807 after retry RPC after error@131166982) -> (continuation of Deferred@390318435 after retry RPC after error@2140935767) -> (continuation of Deferred@1779211738 after retry RPC after error@1716099806) -> (continuation of Deferred@1867009957 after retry RPC after error@1887027893) -> (continuation of Deferred@2123719469 after retry RPC after error@1411422539) -> (continuation of Deferred@2033762573 after retry RPC after error@1206569205)))
21:05:01.484 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:05:01.484 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:05:01.485 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:01.485 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:01.485 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:01.485 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:01.485 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:01.485 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:01.486 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:05:01.504 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:05:01.504 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:05:01.505 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:01.505 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:01.505 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:01.505 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:01.505 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:01.505 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:01.506 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:05:01.524 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:05:01.524 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:05:01.524 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:01.524 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:01.525 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:01.525 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:01.525 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:01.525 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:01.525 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:05:01.544 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:05:01.545 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:05:01.545 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:01.545 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:01.545 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:01.545 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:01.545 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:01.545 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:01.546 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:05:01.564 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:05:01.564 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:05:01.565 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:01.565 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:01.565 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:01.565 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:01.565 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:01.565 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:01.566 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:05:01.584 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:05:01.584 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:05:01.585 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:01.585 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:01.585 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:01.585 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:01.585 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:01.585 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:01.586 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:05:01.604 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:05:01.604 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:05:01.604 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:01.604 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:01.604 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:01.605 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:01.605 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:01.605 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:01.606 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:05:01.664 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:05:01.664 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:05:01.664 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:01.664 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:01.664 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:01.664 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:01.664 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:01.665 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:01.665 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:05:01.666 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=181), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [39ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [41ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [119ms] refreshing cache from master, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [120ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [179ms] refreshing cache from master, [179ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [180ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [180ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [181ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, deferred=Deferred@93429541(state=PENDING, result=null, callback=(continuation of Deferred@923337616 after retry RPC after error@443301698) -> (continuation of Deferred@1091367709 after retry RPC after error@1215112366) -> (continuation of Deferred@931642288 after retry RPC after error@1290409005) -> (continuation of Deferred@1784189622 after retry RPC after error@1261536007) -> (continuation of Deferred@1105325034 after retry RPC after error@1572258951) -> (continuation of Deferred@1943351461 after retry RPC after error@876325987) -> (continuation of Deferred@2051784629 after retry RPC after error@296935047), errback=(continuation of Deferred@923337616 after retry RPC after error@443301698) -> (continuation of Deferred@1091367709 after retry RPC after error@1215112366) -> (continuation of Deferred@931642288 after retry RPC after error@1290409005) -> (continuation of Deferred@1784189622 after retry RPC after error@1261536007) -> (continuation of Deferred@1105325034 after retry RPC after error@1572258951) -> (continuation of Deferred@1943351461 after retry RPC after error@876325987) -> (continuation of Deferred@2051784629 after retry RPC after error@296935047)))
21:05:01.666 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=181), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [39ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [41ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [119ms] refreshing cache from master, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [120ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [179ms] refreshing cache from master, [179ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [180ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [180ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [181ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, deferred=Deferred@93429541(state=PENDING, result=null, callback=(continuation of Deferred@923337616 after retry RPC after error@443301698) -> (continuation of Deferred@1091367709 after retry RPC after error@1215112366) -> (continuation of Deferred@931642288 after retry RPC after error@1290409005) -> (continuation of Deferred@1784189622 after retry RPC after error@1261536007) -> (continuation of Deferred@1105325034 after retry RPC after error@1572258951) -> (continuation of Deferred@1943351461 after retry RPC after error@876325987) -> (continuation of Deferred@2051784629 after retry RPC after error@296935047), errback=(continuation of Deferred@923337616 after retry RPC after error@443301698) -> (continuation of Deferred@1091367709 after retry RPC after error@1215112366) -> (continuation of Deferred@931642288 after retry RPC after error@1290409005) -> (continuation of Deferred@1784189622 after retry RPC after error@1261536007) -> (continuation of Deferred@1105325034 after retry RPC after error@1572258951) -> (continuation of Deferred@1943351461 after retry RPC after error@876325987) -> (continuation of Deferred@2051784629 after retry RPC after error@296935047)))
21:05:01.684 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:05:01.684 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:05:01.685 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:01.685 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:01.685 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:01.685 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:01.685 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:01.685 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:01.686 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:05:01.704 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:05:01.704 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:05:01.705 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:01.705 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:01.705 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:01.705 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:01.705 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:01.705 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:01.706 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:05:01.724 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:05:01.724 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:05:01.724 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:01.725 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:01.725 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:01.725 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:01.725 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:01.725 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:01.726 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:05:01.744 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:05:01.745 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:05:01.745 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:01.745 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:01.745 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:01.745 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:01.745 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:01.745 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:01.747 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:05:01.764 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:05:01.764 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:05:01.765 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:01.765 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:01.765 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:01.765 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:01.765 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:01.765 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:01.766 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:05:01.784 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:05:01.784 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:05:01.785 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:01.785 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:01.785 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:01.785 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:01.785 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:01.785 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:01.786 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:05:01.804 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:05:01.805 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:05:01.805 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:01.805 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:01.805 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:01.806 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:01.806 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:01.806 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:01.807 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:05:01.884 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:05:01.884 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:05:01.885 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:01.885 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:01.885 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:01.885 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:01.885 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:01.885 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:01.886 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:05:01.886 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=201), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [39ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [201ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [201ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [202ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, deferred=Deferred@993813079(state=PENDING, result=null, callback=(continuation of Deferred@422911797 after retry RPC after error@990893095) -> (continuation of Deferred@976748780 after retry RPC after error@1657725734) -> (continuation of Deferred@1395889405 after retry RPC after error@374922320) -> (continuation of Deferred@1813877015 after retry RPC after error@2044692577) -> (continuation of Deferred@914653147 after retry RPC after error@1718525703) -> (continuation of Deferred@1727737844 after retry RPC after error@1667383242) -> (continuation of Deferred@1980624602 after retry RPC after error@1686753723), errback=(continuation of Deferred@422911797 after retry RPC after error@990893095) -> (continuation of Deferred@976748780 after retry RPC after error@1657725734) -> (continuation of Deferred@1395889405 after retry RPC after error@374922320) -> (continuation of Deferred@1813877015 after retry RPC after error@2044692577) -> (continuation of Deferred@914653147 after retry RPC after error@1718525703) -> (continuation of Deferred@1727737844 after retry RPC after error@1667383242) -> (continuation of Deferred@1980624602 after retry RPC after error@1686753723)))
21:05:01.887 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=201), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [39ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [201ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [201ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [202ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, deferred=Deferred@993813079(state=PENDING, result=null, callback=(continuation of Deferred@422911797 after retry RPC after error@990893095) -> (continuation of Deferred@976748780 after retry RPC after error@1657725734) -> (continuation of Deferred@1395889405 after retry RPC after error@374922320) -> (continuation of Deferred@1813877015 after retry RPC after error@2044692577) -> (continuation of Deferred@914653147 after retry RPC after error@1718525703) -> (continuation of Deferred@1727737844 after retry RPC after error@1667383242) -> (continuation of Deferred@1980624602 after retry RPC after error@1686753723), errback=(continuation of Deferred@422911797 after retry RPC after error@990893095) -> (continuation of Deferred@976748780 after retry RPC after error@1657725734) -> (continuation of Deferred@1395889405 after retry RPC after error@374922320) -> (continuation of Deferred@1813877015 after retry RPC after error@2044692577) -> (continuation of Deferred@914653147 after retry RPC after error@1718525703) -> (continuation of Deferred@1727737844 after retry RPC after error@1667383242) -> (continuation of Deferred@1980624602 after retry RPC after error@1686753723)))
21:05:01.905 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:05:01.905 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:05:01.905 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:01.905 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:01.905 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:01.905 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:01.906 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:01.906 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:01.907 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:05:01.924 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:05:01.925 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:05:01.925 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:01.925 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:01.925 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:01.926 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:01.926 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:01.926 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:01.927 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:05:01.944 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:05:01.944 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:05:01.945 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:01.945 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:01.945 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:01.945 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:01.945 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:01.945 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:01.946 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:05:01.964 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:05:01.964 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:05:01.965 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:01.965 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:01.965 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:01.965 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:01.965 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:01.965 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:01.966 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:05:01.984 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:05:01.984 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:05:01.985 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:01.985 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:01.985 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:01.985 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:01.985 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:01.985 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:01.986 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:05:02.004 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:05:02.005 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:05:02.005 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:02.005 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:02.005 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:02.005 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:02.005 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:02.006 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:02.006 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:05:02.084 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:05:02.084 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:05:02.084 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:02.084 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:02.085 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:02.085 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:02.085 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:02.085 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:02.086 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:05:02.144 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:05:02.144 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:05:02.144 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:02.145 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:02.145 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:02.145 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:02.145 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:02.145 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:02.146 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:05:02.146 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=241), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [179ms] refreshing cache from master, [179ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [181ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [181ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [182ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [239ms] refreshing cache from master, [239ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [241ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [241ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [242ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [242ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, deferred=Deferred@1547925494(state=PENDING, result=null, callback=(continuation of Deferred@1683642844 after retry RPC after error@1667931609) -> (continuation of Deferred@1100682768 after retry RPC after error@102769938) -> (continuation of Deferred@805439667 after retry RPC after error@129093439) -> (continuation of Deferred@503538255 after retry RPC after error@420273192) -> (continuation of Deferred@971704362 after retry RPC after error@808116255) -> (continuation of Deferred@893320527 after retry RPC after error@1252825692) -> (continuation of Deferred@736588155 after retry RPC after error@1277838848), errback=(continuation of Deferred@1683642844 after retry RPC after error@1667931609) -> (continuation of Deferred@1100682768 after retry RPC after error@102769938) -> (continuation of Deferred@805439667 after retry RPC after error@129093439) -> (continuation of Deferred@503538255 after retry RPC after error@420273192) -> (continuation of Deferred@971704362 after retry RPC after error@808116255) -> (continuation of Deferred@893320527 after retry RPC after error@1252825692) -> (continuation of Deferred@736588155 after retry RPC after error@1277838848)))
21:05:02.147 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=241), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [179ms] refreshing cache from master, [179ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [181ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [181ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [182ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [239ms] refreshing cache from master, [239ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [241ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [241ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [242ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [242ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, deferred=Deferred@1547925494(state=PENDING, result=null, callback=(continuation of Deferred@1683642844 after retry RPC after error@1667931609) -> (continuation of Deferred@1100682768 after retry RPC after error@102769938) -> (continuation of Deferred@805439667 after retry RPC after error@129093439) -> (continuation of Deferred@503538255 after retry RPC after error@420273192) -> (continuation of Deferred@971704362 after retry RPC after error@808116255) -> (continuation of Deferred@893320527 after retry RPC after error@1252825692) -> (continuation of Deferred@736588155 after retry RPC after error@1277838848), errback=(continuation of Deferred@1683642844 after retry RPC after error@1667931609) -> (continuation of Deferred@1100682768 after retry RPC after error@102769938) -> (continuation of Deferred@805439667 after retry RPC after error@129093439) -> (continuation of Deferred@503538255 after retry RPC after error@420273192) -> (continuation of Deferred@971704362 after retry RPC after error@808116255) -> (continuation of Deferred@893320527 after retry RPC after error@1252825692) -> (continuation of Deferred@736588155 after retry RPC after error@1277838848)))
21:05:02.164 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:05:02.165 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:05:02.165 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:02.165 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:02.166 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:02.166 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:02.166 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:02.166 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:02.166 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:05:02.184 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:05:02.185 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:05:02.185 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:02.185 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:02.185 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:02.185 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:02.185 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:02.185 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:02.186 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:05:02.204 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:05:02.204 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:05:02.205 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:02.205 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:02.205 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:02.205 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:02.205 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:02.205 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:02.206 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:05:02.224 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:05:02.224 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:05:02.225 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:02.225 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:02.225 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:02.225 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:02.225 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:02.225 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:02.226 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:05:02.244 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:05:02.245 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:05:02.245 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:02.245 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:02.245 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:02.245 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:02.245 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:02.245 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:02.246 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:05:02.264 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:05:02.264 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:05:02.265 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:02.265 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:02.265 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:02.265 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:02.265 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:02.265 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:02.266 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:05:02.284 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:05:02.284 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:05:02.285 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:02.285 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:02.285 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:02.285 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:02.285 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:02.285 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:02.286 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:05:02.364 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:05:02.365 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:05:02.365 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:02.365 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:02.365 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:02.365 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:02.365 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:02.365 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:02.366 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:05:02.367 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=202), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [201ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [201ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [202ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, deferred=Deferred@37302019(state=PENDING, result=null, callback=(continuation of Deferred@733639654 after retry RPC after error@725591758) -> (continuation of Deferred@461997023 after retry RPC after error@1350825626) -> (continuation of Deferred@685677103 after retry RPC after error@2122399788) -> (continuation of Deferred@692167512 after retry RPC after error@1715058636) -> (continuation of Deferred@988436166 after retry RPC after error@1310646679) -> (continuation of Deferred@1784791151 after retry RPC after error@1095601464) -> (continuation of Deferred@704233393 after retry RPC after error@282255680), errback=(continuation of Deferred@733639654 after retry RPC after error@725591758) -> (continuation of Deferred@461997023 after retry RPC after error@1350825626) -> (continuation of Deferred@685677103 after retry RPC after error@2122399788) -> (continuation of Deferred@692167512 after retry RPC after error@1715058636) -> (continuation of Deferred@988436166 after retry RPC after error@1310646679) -> (continuation of Deferred@1784791151 after retry RPC after error@1095601464) -> (continuation of Deferred@704233393 after retry RPC after error@282255680)))
21:05:02.367 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=202), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [201ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [201ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [202ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, deferred=Deferred@37302019(state=PENDING, result=null, callback=(continuation of Deferred@733639654 after retry RPC after error@725591758) -> (continuation of Deferred@461997023 after retry RPC after error@1350825626) -> (continuation of Deferred@685677103 after retry RPC after error@2122399788) -> (continuation of Deferred@692167512 after retry RPC after error@1715058636) -> (continuation of Deferred@988436166 after retry RPC after error@1310646679) -> (continuation of Deferred@1784791151 after retry RPC after error@1095601464) -> (continuation of Deferred@704233393 after retry RPC after error@282255680), errback=(continuation of Deferred@733639654 after retry RPC after error@725591758) -> (continuation of Deferred@461997023 after retry RPC after error@1350825626) -> (continuation of Deferred@685677103 after retry RPC after error@2122399788) -> (continuation of Deferred@692167512 after retry RPC after error@1715058636) -> (continuation of Deferred@988436166 after retry RPC after error@1310646679) -> (continuation of Deferred@1784791151 after retry RPC after error@1095601464) -> (continuation of Deferred@704233393 after retry RPC after error@282255680)))
21:05:02.384 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:05:02.384 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:05:02.385 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:02.386 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:02.386 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:02.386 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:02.386 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:02.386 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:02.387 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:05:02.404 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:05:02.404 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:05:02.405 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:02.405 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:02.405 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:02.405 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:02.405 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:02.405 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:02.406 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:05:02.424 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:05:02.424 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:05:02.424 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:02.424 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:02.424 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:02.425 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:02.425 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:02.425 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:02.426 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:05:02.444 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:05:02.445 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:05:02.445 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:02.445 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:02.445 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:02.445 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:02.445 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:02.445 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:02.446 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:05:02.464 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:05:02.464 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:05:02.465 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:02.465 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:02.465 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:02.465 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:02.465 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:02.465 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:02.466 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:05:02.504 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:05:02.504 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:05:02.505 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:02.505 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:02.505 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:02.505 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:02.505 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:02.505 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:02.506 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:05:02.564 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:05:02.565 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:05:02.565 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:02.565 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:02.565 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:02.565 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:02.565 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:02.565 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:02.566 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:05:02.604 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:05:02.605 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:05:02.605 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:02.605 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:02.605 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:02.605 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:02.606 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:02.606 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:02.606 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:05:02.607 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=222), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [39ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [181ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [181ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [182ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [222ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [222ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [222ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [223ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, deferred=Deferred@642146171(state=PENDING, result=null, callback=(continuation of Deferred@442901482 after retry RPC after error@894800207) -> (continuation of Deferred@567156678 after retry RPC after error@904343811) -> (continuation of Deferred@515194150 after retry RPC after error@133367894) -> (continuation of Deferred@1444022081 after retry RPC after error@1517374377) -> (continuation of Deferred@1838813644 after retry RPC after error@1916011921) -> (continuation of Deferred@886717593 after retry RPC after error@1680217512) -> (continuation of Deferred@1494183057 after retry RPC after error@189129388), errback=(continuation of Deferred@442901482 after retry RPC after error@894800207) -> (continuation of Deferred@567156678 after retry RPC after error@904343811) -> (continuation of Deferred@515194150 after retry RPC after error@133367894) -> (continuation of Deferred@1444022081 after retry RPC after error@1517374377) -> (continuation of Deferred@1838813644 after retry RPC after error@1916011921) -> (continuation of Deferred@886717593 after retry RPC after error@1680217512) -> (continuation of Deferred@1494183057 after retry RPC after error@189129388)))
21:05:02.607 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=222), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [39ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [181ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [181ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [182ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [222ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [222ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [222ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [223ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, deferred=Deferred@642146171(state=PENDING, result=null, callback=(continuation of Deferred@442901482 after retry RPC after error@894800207) -> (continuation of Deferred@567156678 after retry RPC after error@904343811) -> (continuation of Deferred@515194150 after retry RPC after error@133367894) -> (continuation of Deferred@1444022081 after retry RPC after error@1517374377) -> (continuation of Deferred@1838813644 after retry RPC after error@1916011921) -> (continuation of Deferred@886717593 after retry RPC after error@1680217512) -> (continuation of Deferred@1494183057 after retry RPC after error@189129388), errback=(continuation of Deferred@442901482 after retry RPC after error@894800207) -> (continuation of Deferred@567156678 after retry RPC after error@904343811) -> (continuation of Deferred@515194150 after retry RPC after error@133367894) -> (continuation of Deferred@1444022081 after retry RPC after error@1517374377) -> (continuation of Deferred@1838813644 after retry RPC after error@1916011921) -> (continuation of Deferred@886717593 after retry RPC after error@1680217512) -> (continuation of Deferred@1494183057 after retry RPC after error@189129388)))
21:05:02.624 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:05:02.624 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:05:02.625 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:02.625 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:02.625 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:02.625 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:02.625 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:02.625 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:02.626 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:05:02.644 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:05:02.644 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:05:02.645 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:02.645 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:02.645 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:02.645 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:02.645 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:02.645 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:02.646 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:05:02.664 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:05:02.664 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:05:02.665 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:02.665 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:02.665 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:02.665 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:02.665 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:02.665 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:02.666 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:05:02.685 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:05:02.685 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:05:02.685 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:02.685 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:02.685 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:02.685 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:02.686 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:02.686 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:02.687 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:05:02.704 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:05:02.705 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:05:02.705 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:02.705 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:02.705 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:02.705 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:02.705 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:02.705 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:02.706 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:05:02.744 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:05:02.744 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:05:02.745 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:02.745 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:02.745 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:02.745 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:02.745 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:02.745 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:02.746 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:05:02.804 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:05:02.804 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:05:02.805 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:02.805 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:02.805 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:02.805 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:02.805 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:02.805 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:02.806 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:05:02.885 [DEBUG - pool-40-thread-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=260), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [181ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [181ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [182ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, deferred=Deferred@177834793(state=PENDING, result=null, callback=(continuation of Deferred@883285898 after retry RPC after error@328828199) -> (continuation of Deferred@307336794 after retry RPC after error@1415026682) -> (continuation of Deferred@1318345589 after retry RPC after error@293297981) -> (continuation of Deferred@252114958 after retry RPC after error@1736853075) -> (continuation of Deferred@187033125 after retry RPC after error@1080873675) -> (continuation of Deferred@2014479424 after retry RPC after error@762569167) -> (continuation of Deferred@155397770 after retry RPC after error@790581219), errback=(continuation of Deferred@883285898 after retry RPC after error@328828199) -> (continuation of Deferred@307336794 after retry RPC after error@1415026682) -> (continuation of Deferred@1318345589 after retry RPC after error@293297981) -> (continuation of Deferred@252114958 after retry RPC after error@1736853075) -> (continuation of Deferred@187033125 after retry RPC after error@1080873675) -> (continuation of Deferred@2014479424 after retry RPC after error@762569167) -> (continuation of Deferred@155397770 after retry RPC after error@790581219)))
21:05:02.885 [WARN - pool-40-thread-1] (HashedWheelTimer.java:716) An exception was thrown by TimerTask.
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343) ~[main/:?]
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at java.lang.Thread.run(Thread.java:748) [?:1.8.0_275]
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86)
at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106)
at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198)
at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158)
at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953)
at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923)
at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464)
at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713)
at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701)
at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788)
at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501)
at java.lang.Thread.run(Thread.java:748)
21:05:02.904 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:05:02.904 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:05:02.905 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:02.905 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:02.905 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:02.905 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:02.905 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:02.905 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:02.906 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:05:02.924 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:05:02.924 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:05:02.925 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:02.925 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:02.925 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:02.925 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:02.925 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:02.925 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:02.926 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:05:02.944 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:05:02.944 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:05:02.944 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:02.944 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:02.945 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:02.945 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:02.945 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:02.945 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:02.946 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:05:02.964 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:05:02.965 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:05:02.965 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:02.965 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:02.965 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:02.965 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:02.965 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:02.965 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:02.966 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:05:02.984 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:05:02.984 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:05:02.985 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:02.985 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:02.985 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:02.985 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:02.985 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:02.985 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:02.986 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:05:03.024 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:05:03.024 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:05:03.025 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:03.025 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:03.025 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:03.025 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:03.025 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:03.025 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:03.026 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:05:03.044 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:05:03.044 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:05:03.044 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:03.044 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:03.044 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:03.045 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:03.045 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:03.045 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:03.046 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:05:03.064 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:05:03.065 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:05:03.065 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:03.065 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:03.065 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:03.065 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:03.065 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:03.065 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:03.066 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:05:03.067 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=162), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [39ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [139ms] refreshing cache from master, [139ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [141ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [141ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [141ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [161ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [161ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [162ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [163ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, deferred=Deferred@502525295(state=PENDING, result=null, callback=(continuation of Deferred@707283602 after retry RPC after error@1212939889) -> (continuation of Deferred@33832994 after retry RPC after error@10374552) -> (continuation of Deferred@1745546179 after retry RPC after error@2141534405) -> (continuation of Deferred@1602253379 after retry RPC after error@763369126) -> (continuation of Deferred@309048358 after retry RPC after error@1155454398) -> (continuation of Deferred@403744012 after retry RPC after error@889759118) -> (continuation of Deferred@2047737229 after retry RPC after error@1221370967), errback=(continuation of Deferred@707283602 after retry RPC after error@1212939889) -> (continuation of Deferred@33832994 after retry RPC after error@10374552) -> (continuation of Deferred@1745546179 after retry RPC after error@2141534405) -> (continuation of Deferred@1602253379 after retry RPC after error@763369126) -> (continuation of Deferred@309048358 after retry RPC after error@1155454398) -> (continuation of Deferred@403744012 after retry RPC after error@889759118) -> (continuation of Deferred@2047737229 after retry RPC after error@1221370967)))
21:05:03.067 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=162), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [39ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [139ms] refreshing cache from master, [139ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [141ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [141ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [141ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [161ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [161ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [162ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [163ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, deferred=Deferred@502525295(state=PENDING, result=null, callback=(continuation of Deferred@707283602 after retry RPC after error@1212939889) -> (continuation of Deferred@33832994 after retry RPC after error@10374552) -> (continuation of Deferred@1745546179 after retry RPC after error@2141534405) -> (continuation of Deferred@1602253379 after retry RPC after error@763369126) -> (continuation of Deferred@309048358 after retry RPC after error@1155454398) -> (continuation of Deferred@403744012 after retry RPC after error@889759118) -> (continuation of Deferred@2047737229 after retry RPC after error@1221370967), errback=(continuation of Deferred@707283602 after retry RPC after error@1212939889) -> (continuation of Deferred@33832994 after retry RPC after error@10374552) -> (continuation of Deferred@1745546179 after retry RPC after error@2141534405) -> (continuation of Deferred@1602253379 after retry RPC after error@763369126) -> (continuation of Deferred@309048358 after retry RPC after error@1155454398) -> (continuation of Deferred@403744012 after retry RPC after error@889759118) -> (continuation of Deferred@2047737229 after retry RPC after error@1221370967)))
21:05:03.084 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:05:03.084 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:05:03.085 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:03.085 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:03.085 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:03.085 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:03.085 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:03.085 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:03.086 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:05:03.104 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:05:03.104 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:05:03.104 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:03.104 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:03.104 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:03.105 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:03.105 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:03.105 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:03.105 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:05:03.124 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:05:03.124 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:05:03.125 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:03.125 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:03.125 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:03.125 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:03.125 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:03.125 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:03.126 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:05:03.144 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:05:03.144 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:05:03.145 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:03.145 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:03.145 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:03.145 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:03.145 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:03.145 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:03.146 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:05:03.164 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:05:03.164 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:05:03.164 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:03.164 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:03.165 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:03.165 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:03.165 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:03.165 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:03.166 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:05:03.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:05:03.197515 23569 master_proxy_rpc.cc:177] Failed to determine new leader Master: Timed out: ConnectToClusterRpc(addrs: 127.22.168.62:37239,127.22.168.61:32947,127.22.168.60:33247, num_attempts: 97) passed its deadline: Not found: no leader found: ConnectToClusterRpc(addrs: 127.22.168.62:37239,127.22.168.61:32947,127.22.168.60:33247, num_attempts: 1)
21:05:03.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:05:03.199952 23569 meta_cache.cc:788] Timed out: LookupRpcById { tablet: '01aa0e192c434b468ec7ed90d48563b3', attempt: 2 } failed: LookupRpcById timed out after deadline expired: LookupRpcById { tablet: '01aa0e192c434b468ec7ed90d48563b3', attempt: 2 } passed its deadline: Timed out: ConnectToClusterRpc(addrs: 127.22.168.62:37239,127.22.168.61:32947,127.22.168.60:33247, num_attempts: 97) passed its deadline: Not found: no leader found: ConnectToClusterRpc(addrs: 127.22.168.62:37239,127.22.168.61:32947,127.22.168.60:33247, num_attempts: 1)
21:05:03.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:05:03.201997 23569 master_proxy_rpc.cc:203] Re-attempting LookupRpcById request to leader Master (127.22.168.60:33247)
21:05:03.204 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:05:03.205 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:05:03.205 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:03.205 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:03.205 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:03.205 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:03.205 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:03.206 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:03.207 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:05:03.244 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:05:03.245 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:05:03.245 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:03.245 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:03.245 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:03.245 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:03.245 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:03.245 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:03.247 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:05:03.264 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:05:03.264 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:05:03.265 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:03.265 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:03.265 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:03.265 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:03.265 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:03.265 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:03.266 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:05:03.267 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=182), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [19ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [21ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [79ms] refreshing cache from master, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [123ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [161ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [161ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [163ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [163ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [181ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [181ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [182ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, deferred=Deferred@840557073(state=PENDING, result=null, callback=(continuation of Deferred@2125416914 after retry RPC after error@55144537) -> (continuation of Deferred@48833382 after retry RPC after error@2141016128) -> (continuation of Deferred@2071492536 after retry RPC after error@863705910) -> (continuation of Deferred@2017655852 after retry RPC after error@1833479220) -> (continuation of Deferred@1437975442 after retry RPC after error@1056755897) -> (continuation of Deferred@1370548438 after retry RPC after error@2033309656) -> (continuation of Deferred@48439871 after retry RPC after error@1514292162), errback=(continuation of Deferred@2125416914 after retry RPC after error@55144537) -> (continuation of Deferred@48833382 after retry RPC after error@2141016128) -> (continuation of Deferred@2071492536 after retry RPC after error@863705910) -> (continuation of Deferred@2017655852 after retry RPC after error@1833479220) -> (continuation of Deferred@1437975442 after retry RPC after error@1056755897) -> (continuation of Deferred@1370548438 after retry RPC after error@2033309656) -> (continuation of Deferred@48439871 after retry RPC after error@1514292162)))
21:05:03.267 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=182), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [19ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [21ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [79ms] refreshing cache from master, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [123ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [161ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [161ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [163ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [163ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [181ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [181ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [182ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, deferred=Deferred@840557073(state=PENDING, result=null, callback=(continuation of Deferred@2125416914 after retry RPC after error@55144537) -> (continuation of Deferred@48833382 after retry RPC after error@2141016128) -> (continuation of Deferred@2071492536 after retry RPC after error@863705910) -> (continuation of Deferred@2017655852 after retry RPC after error@1833479220) -> (continuation of Deferred@1437975442 after retry RPC after error@1056755897) -> (continuation of Deferred@1370548438 after retry RPC after error@2033309656) -> (continuation of Deferred@48439871 after retry RPC after error@1514292162), errback=(continuation of Deferred@2125416914 after retry RPC after error@55144537) -> (continuation of Deferred@48833382 after retry RPC after error@2141016128) -> (continuation of Deferred@2071492536 after retry RPC after error@863705910) -> (continuation of Deferred@2017655852 after retry RPC after error@1833479220) -> (continuation of Deferred@1437975442 after retry RPC after error@1056755897) -> (continuation of Deferred@1370548438 after retry RPC after error@2033309656) -> (continuation of Deferred@48439871 after retry RPC after error@1514292162)))
21:05:03.284 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:05:03.285 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:03.285 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:03.285 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:03.285 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:05:03.286 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:03.286 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:03.286 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:03.286 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:05:03.304 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:05:03.304 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:05:03.304 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:03.304 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:03.305 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:03.305 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:03.305 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:03.305 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:03.306 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:05:03.324 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:05:03.325 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:05:03.325 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:03.325 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:03.325 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:03.325 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:03.325 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:03.325 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:03.326 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:05:03.344 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:05:03.344 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:05:03.345 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:03.345 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:03.345 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:03.345 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:03.345 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:03.345 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:03.346 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:05:03.364 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:05:03.364 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:05:03.364 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:03.365 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:03.365 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:03.365 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:03.365 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:03.365 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:03.366 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:05:03.384 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:05:03.385 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:05:03.385 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:03.385 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:03.385 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:03.385 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:03.386 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:03.386 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:03.386 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:05:03.424 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:05:03.425 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:05:03.425 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:03.425 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:03.425 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:03.425 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:03.425 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:03.425 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:03.426 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:05:03.504 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:05:03.504 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:05:03.505 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:03.505 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:03.505 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:03.505 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:03.505 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:03.505 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:03.506 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:05:03.524 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:05:03.525 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:05:03.525 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:03.525 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:03.525 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:03.525 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:03.525 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:03.525 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:03.526 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:05:03.527 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=9, TimeoutTracker(timeout=250, elapsed=242), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [2ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [19ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [141ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [141ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [142ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [221ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [221ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [221ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [222ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [241ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [241ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [242ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [242ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, deferred=Deferred@1210203399(state=PENDING, result=null, callback=(continuation of Deferred@292673388 after retry RPC after error@698344131) -> (continuation of Deferred@2119248258 after retry RPC after error@658696522) -> (continuation of Deferred@941064313 after retry RPC after error@1850274882) -> (continuation of Deferred@430099760 after retry RPC after error@689602551) -> (continuation of Deferred@764032741 after retry RPC after error@80297458) -> (continuation of Deferred@185445142 after retry RPC after error@1905432647) -> (continuation of Deferred@231974286 after retry RPC after error@1077428075) -> (continuation of Deferred@646674785 after retry RPC after error@1136565591), errback=(continuation of Deferred@292673388 after retry RPC after error@698344131) -> (continuation of Deferred@2119248258 after retry RPC after error@658696522) -> (continuation of Deferred@941064313 after retry RPC after error@1850274882) -> (continuation of Deferred@430099760 after retry RPC after error@689602551) -> (continuation of Deferred@764032741 after retry RPC after error@80297458) -> (continuation of Deferred@185445142 after retry RPC after error@1905432647) -> (continuation of Deferred@231974286 after retry RPC after error@1077428075) -> (continuation of Deferred@646674785 after retry RPC after error@1136565591)))
21:05:03.527 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=9, TimeoutTracker(timeout=250, elapsed=242), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [2ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [19ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [141ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [141ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [142ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [221ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [221ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [221ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [222ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [241ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [241ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [242ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [242ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, deferred=Deferred@1210203399(state=PENDING, result=null, callback=(continuation of Deferred@292673388 after retry RPC after error@698344131) -> (continuation of Deferred@2119248258 after retry RPC after error@658696522) -> (continuation of Deferred@941064313 after retry RPC after error@1850274882) -> (continuation of Deferred@430099760 after retry RPC after error@689602551) -> (continuation of Deferred@764032741 after retry RPC after error@80297458) -> (continuation of Deferred@185445142 after retry RPC after error@1905432647) -> (continuation of Deferred@231974286 after retry RPC after error@1077428075) -> (continuation of Deferred@646674785 after retry RPC after error@1136565591), errback=(continuation of Deferred@292673388 after retry RPC after error@698344131) -> (continuation of Deferred@2119248258 after retry RPC after error@658696522) -> (continuation of Deferred@941064313 after retry RPC after error@1850274882) -> (continuation of Deferred@430099760 after retry RPC after error@689602551) -> (continuation of Deferred@764032741 after retry RPC after error@80297458) -> (continuation of Deferred@185445142 after retry RPC after error@1905432647) -> (continuation of Deferred@231974286 after retry RPC after error@1077428075) -> (continuation of Deferred@646674785 after retry RPC after error@1136565591)))
21:05:03.544 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:05:03.544 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:05:03.545 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:03.545 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:03.545 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:03.545 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:03.545 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:03.545 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:03.546 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:05:03.564 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:05:03.564 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:05:03.565 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:03.565 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:03.565 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:03.565 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:03.565 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:03.565 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:03.566 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:05:03.584 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:05:03.584 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:05:03.584 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:03.584 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:03.585 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:03.585 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:03.585 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:03.585 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:03.586 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:05:03.604 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:05:03.605 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:05:03.605 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:03.605 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:03.605 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:03.605 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:03.605 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:03.605 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:03.606 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:05:03.624 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:05:03.624 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:05:03.625 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:03.625 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:03.625 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:03.625 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:03.626 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:03.626 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:03.626 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:05:03.644 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:05:03.644 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:05:03.645 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:03.645 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:03.645 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:03.645 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:03.645 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:03.645 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:03.646 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:05:03.684 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:05:03.684 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:05:03.684 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:03.685 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:03.685 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:03.685 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:03.685 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:03.685 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:03.686 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:05:03.805 [DEBUG - pool-40-thread-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=260), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [39ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [141ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [141ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [142ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, deferred=Deferred@821192790(state=PENDING, result=null, callback=(continuation of Deferred@1056944336 after retry RPC after error@2107546637) -> (continuation of Deferred@52445309 after retry RPC after error@548612247) -> (continuation of Deferred@1661560161 after retry RPC after error@1795093497) -> (continuation of Deferred@1809610099 after retry RPC after error@1385307919) -> (continuation of Deferred@2048412241 after retry RPC after error@1743003600) -> (continuation of Deferred@1864079794 after retry RPC after error@758356281) -> (continuation of Deferred@364975116 after retry RPC after error@1377600507), errback=(continuation of Deferred@1056944336 after retry RPC after error@2107546637) -> (continuation of Deferred@52445309 after retry RPC after error@548612247) -> (continuation of Deferred@1661560161 after retry RPC after error@1795093497) -> (continuation of Deferred@1809610099 after retry RPC after error@1385307919) -> (continuation of Deferred@2048412241 after retry RPC after error@1743003600) -> (continuation of Deferred@1864079794 after retry RPC after error@758356281) -> (continuation of Deferred@364975116 after retry RPC after error@1377600507)))
21:05:03.805 [WARN - pool-40-thread-1] (HashedWheelTimer.java:716) An exception was thrown by TimerTask.
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343) ~[main/:?]
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at java.lang.Thread.run(Thread.java:748) [?:1.8.0_275]
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86)
at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106)
at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198)
at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158)
at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953)
at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923)
at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464)
at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713)
at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701)
at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788)
at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501)
at java.lang.Thread.run(Thread.java:748)
21:05:03.824 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:05:03.824 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:05:03.825 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:03.825 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:03.825 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:03.825 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:03.825 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:03.825 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:03.826 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:05:03.844 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:05:03.844 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:05:03.845 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:03.845 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:03.845 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:03.845 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:03.845 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:03.845 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:03.845 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:05:03.864 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:05:03.865 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:05:03.865 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:03.865 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:03.865 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:03.865 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:03.865 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:03.865 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:03.866 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:05:03.884 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:05:03.884 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:05:03.885 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:03.885 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:03.885 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:03.885 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:03.885 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:03.885 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:03.886 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:05:03.904 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:05:03.904 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:05:03.905 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:03.905 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:03.905 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:03.905 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:03.905 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:03.905 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:03.906 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:05:03.925 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:05:03.925 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:05:03.925 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:03.925 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:03.925 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:03.925 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:03.926 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:03.926 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:03.927 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:05:03.944 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:05:03.944 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:05:03.945 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:03.945 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:03.945 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:03.945 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:03.945 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:03.945 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:03.946 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:05:04.004 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:05:04.004 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:05:04.005 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:04.005 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:04.005 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:04.005 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:04.005 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:04.005 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:04.007 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:05:04.024 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:05:04.024 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:05:04.025 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:04.025 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:04.025 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:04.025 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:04.025 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:04.025 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:04.026 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:05:04.026 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=202), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [201ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [201ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [202ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, deferred=Deferred@340259104(state=PENDING, result=null, callback=(continuation of Deferred@103210913 after retry RPC after error@1058702956) -> (continuation of Deferred@646114791 after retry RPC after error@233293542) -> (continuation of Deferred@1032637228 after retry RPC after error@1456345879) -> (continuation of Deferred@1737634300 after retry RPC after error@1416698654) -> (continuation of Deferred@260009751 after retry RPC after error@697271108) -> (continuation of Deferred@1600408152 after retry RPC after error@51450689) -> (continuation of Deferred@2002961379 after retry RPC after error@581106664), errback=(continuation of Deferred@103210913 after retry RPC after error@1058702956) -> (continuation of Deferred@646114791 after retry RPC after error@233293542) -> (continuation of Deferred@1032637228 after retry RPC after error@1456345879) -> (continuation of Deferred@1737634300 after retry RPC after error@1416698654) -> (continuation of Deferred@260009751 after retry RPC after error@697271108) -> (continuation of Deferred@1600408152 after retry RPC after error@51450689) -> (continuation of Deferred@2002961379 after retry RPC after error@581106664)))
21:05:04.027 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=202), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [201ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [201ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [202ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, deferred=Deferred@340259104(state=PENDING, result=null, callback=(continuation of Deferred@103210913 after retry RPC after error@1058702956) -> (continuation of Deferred@646114791 after retry RPC after error@233293542) -> (continuation of Deferred@1032637228 after retry RPC after error@1456345879) -> (continuation of Deferred@1737634300 after retry RPC after error@1416698654) -> (continuation of Deferred@260009751 after retry RPC after error@697271108) -> (continuation of Deferred@1600408152 after retry RPC after error@51450689) -> (continuation of Deferred@2002961379 after retry RPC after error@581106664), errback=(continuation of Deferred@103210913 after retry RPC after error@1058702956) -> (continuation of Deferred@646114791 after retry RPC after error@233293542) -> (continuation of Deferred@1032637228 after retry RPC after error@1456345879) -> (continuation of Deferred@1737634300 after retry RPC after error@1416698654) -> (continuation of Deferred@260009751 after retry RPC after error@697271108) -> (continuation of Deferred@1600408152 after retry RPC after error@51450689) -> (continuation of Deferred@2002961379 after retry RPC after error@581106664)))
21:05:04.045 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:05:04.045 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:05:04.045 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:04.045 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:04.045 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:04.045 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:04.046 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:04.046 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:04.047 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:05:04.064 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:05:04.064 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:05:04.065 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:04.065 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:04.065 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:04.065 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:04.065 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:04.065 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:04.066 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:05:04.084 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:05:04.084 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:05:04.085 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:04.085 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:04.085 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:04.085 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:04.085 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:04.085 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:04.086 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:05:04.104 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:05:04.104 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:05:04.105 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:04.105 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:04.105 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:04.105 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:04.105 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:04.105 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:04.106 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:05:04.124 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:05:04.125 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:05:04.125 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:04.125 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:04.125 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:04.125 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:04.125 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:04.125 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:04.126 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:05:04.164 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:05:04.165 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:05:04.165 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:04.165 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:04.165 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:04.165 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:04.165 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:04.165 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:04.166 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:05:04.224 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:05:04.224 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:05:04.225 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:04.225 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:04.225 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:04.225 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:04.225 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:04.225 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:04.226 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:05:04.284 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:05:04.284 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:05:04.285 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:04.285 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:04.285 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:04.285 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:04.285 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:04.285 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:04.286 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:05:04.287 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=241), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [59ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [181ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [181ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [182ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [241ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [241ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [242ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [242ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, deferred=Deferred@36714630(state=PENDING, result=null, callback=(continuation of Deferred@1058724242 after retry RPC after error@1175932899) -> (continuation of Deferred@1076348037 after retry RPC after error@1137717263) -> (continuation of Deferred@711960494 after retry RPC after error@1459330391) -> (continuation of Deferred@679897313 after retry RPC after error@1779721654) -> (continuation of Deferred@1024585101 after retry RPC after error@923459165) -> (continuation of Deferred@743471421 after retry RPC after error@1705017556) -> (continuation of Deferred@1466922088 after retry RPC after error@922132718), errback=(continuation of Deferred@1058724242 after retry RPC after error@1175932899) -> (continuation of Deferred@1076348037 after retry RPC after error@1137717263) -> (continuation of Deferred@711960494 after retry RPC after error@1459330391) -> (continuation of Deferred@679897313 after retry RPC after error@1779721654) -> (continuation of Deferred@1024585101 after retry RPC after error@923459165) -> (continuation of Deferred@743471421 after retry RPC after error@1705017556) -> (continuation of Deferred@1466922088 after retry RPC after error@922132718)))
21:05:04.287 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=241), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [59ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [181ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [181ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [182ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [241ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [241ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [242ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [242ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, deferred=Deferred@36714630(state=PENDING, result=null, callback=(continuation of Deferred@1058724242 after retry RPC after error@1175932899) -> (continuation of Deferred@1076348037 after retry RPC after error@1137717263) -> (continuation of Deferred@711960494 after retry RPC after error@1459330391) -> (continuation of Deferred@679897313 after retry RPC after error@1779721654) -> (continuation of Deferred@1024585101 after retry RPC after error@923459165) -> (continuation of Deferred@743471421 after retry RPC after error@1705017556) -> (continuation of Deferred@1466922088 after retry RPC after error@922132718), errback=(continuation of Deferred@1058724242 after retry RPC after error@1175932899) -> (continuation of Deferred@1076348037 after retry RPC after error@1137717263) -> (continuation of Deferred@711960494 after retry RPC after error@1459330391) -> (continuation of Deferred@679897313 after retry RPC after error@1779721654) -> (continuation of Deferred@1024585101 after retry RPC after error@923459165) -> (continuation of Deferred@743471421 after retry RPC after error@1705017556) -> (continuation of Deferred@1466922088 after retry RPC after error@922132718)))
21:05:04.304 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:05:04.304 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:05:04.305 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:04.305 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:04.305 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:04.305 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:04.305 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:04.305 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:04.306 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:05:04.324 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:05:04.324 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:05:04.324 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:04.324 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:04.325 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:04.325 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:04.325 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:04.325 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:04.326 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:05:04.344 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:05:04.345 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:05:04.345 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:04.345 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:04.345 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:04.345 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:04.345 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:04.345 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:04.346 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:05:04.364 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:05:04.364 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:05:04.365 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:04.365 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:04.365 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:04.365 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:04.365 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:04.365 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:04.366 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:05:04.384 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:05:04.384 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:05:04.384 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:04.384 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:04.384 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:04.384 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:04.385 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:04.385 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:04.385 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:05:04.404 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:05:04.404 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:05:04.405 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:04.405 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:04.405 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:04.405 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:04.405 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:04.405 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:04.406 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:05:04.464 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:05:04.464 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:05:04.465 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:04.465 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:04.465 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:04.465 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:04.465 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:04.465 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:04.466 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:05:04.466 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=161), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [79ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [80ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [161ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [161ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [161ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, deferred=Deferred@1199493901(state=PENDING, result=null, callback=(continuation of Deferred@1470347815 after retry RPC after error@266336108) -> (continuation of Deferred@341843236 after retry RPC after error@624382494) -> (continuation of Deferred@1871685149 after retry RPC after error@530270231) -> (continuation of Deferred@193392637 after retry RPC after error@348056369) -> (continuation of Deferred@110299381 after retry RPC after error@1124934592) -> (continuation of Deferred@2143868251 after retry RPC after error@474767088), errback=(continuation of Deferred@1470347815 after retry RPC after error@266336108) -> (continuation of Deferred@341843236 after retry RPC after error@624382494) -> (continuation of Deferred@1871685149 after retry RPC after error@530270231) -> (continuation of Deferred@193392637 after retry RPC after error@348056369) -> (continuation of Deferred@110299381 after retry RPC after error@1124934592) -> (continuation of Deferred@2143868251 after retry RPC after error@474767088)))
21:05:04.466 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=161), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [79ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [80ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [161ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [161ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [161ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, deferred=Deferred@1199493901(state=PENDING, result=null, callback=(continuation of Deferred@1470347815 after retry RPC after error@266336108) -> (continuation of Deferred@341843236 after retry RPC after error@624382494) -> (continuation of Deferred@1871685149 after retry RPC after error@530270231) -> (continuation of Deferred@193392637 after retry RPC after error@348056369) -> (continuation of Deferred@110299381 after retry RPC after error@1124934592) -> (continuation of Deferred@2143868251 after retry RPC after error@474767088), errback=(continuation of Deferred@1470347815 after retry RPC after error@266336108) -> (continuation of Deferred@341843236 after retry RPC after error@624382494) -> (continuation of Deferred@1871685149 after retry RPC after error@530270231) -> (continuation of Deferred@193392637 after retry RPC after error@348056369) -> (continuation of Deferred@110299381 after retry RPC after error@1124934592) -> (continuation of Deferred@2143868251 after retry RPC after error@474767088)))
21:05:04.484 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:05:04.484 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:05:04.485 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:04.485 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:04.485 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:04.485 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:04.485 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:04.485 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:04.486 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:05:04.505 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:05:04.505 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:05:04.505 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:04.505 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:04.506 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:04.506 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:04.506 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:04.506 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:04.507 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:05:04.524 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:05:04.525 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:05:04.525 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:04.525 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:04.525 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:04.525 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:04.525 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:04.525 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:04.526 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:05:04.544 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:05:04.544 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:05:04.545 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:04.545 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:04.545 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:04.545 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:04.545 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:04.545 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:04.546 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:05:04.564 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:05:04.564 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:05:04.565 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:04.565 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:04.565 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:04.565 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:04.565 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:04.565 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:04.566 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:05:04.604 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:05:04.604 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:05:04.605 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:04.605 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:04.605 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:04.605 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:04.605 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:04.605 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:04.606 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:05:04.684 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:05:04.684 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:05:04.684 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:04.684 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:04.684 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:04.684 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:04.685 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:04.685 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:04.686 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:05:04.686 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=202), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [23ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [199ms] refreshing cache from master, [199ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [200ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [201ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [201ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, deferred=Deferred@1477848950(state=PENDING, result=null, callback=(continuation of Deferred@19284714 after retry RPC after error@1857876693) -> (continuation of Deferred@2061251002 after retry RPC after error@324586504) -> (continuation of Deferred@542834075 after retry RPC after error@695834911) -> (continuation of Deferred@850607211 after retry RPC after error@1742638777) -> (continuation of Deferred@270471676 after retry RPC after error@1976288129) -> (continuation of Deferred@1580267831 after retry RPC after error@1292373774), errback=(continuation of Deferred@19284714 after retry RPC after error@1857876693) -> (continuation of Deferred@2061251002 after retry RPC after error@324586504) -> (continuation of Deferred@542834075 after retry RPC after error@695834911) -> (continuation of Deferred@850607211 after retry RPC after error@1742638777) -> (continuation of Deferred@270471676 after retry RPC after error@1976288129) -> (continuation of Deferred@1580267831 after retry RPC after error@1292373774)))
21:05:04.687 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=202), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [23ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [199ms] refreshing cache from master, [199ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [200ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [201ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [201ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, deferred=Deferred@1477848950(state=PENDING, result=null, callback=(continuation of Deferred@19284714 after retry RPC after error@1857876693) -> (continuation of Deferred@2061251002 after retry RPC after error@324586504) -> (continuation of Deferred@542834075 after retry RPC after error@695834911) -> (continuation of Deferred@850607211 after retry RPC after error@1742638777) -> (continuation of Deferred@270471676 after retry RPC after error@1976288129) -> (continuation of Deferred@1580267831 after retry RPC after error@1292373774), errback=(continuation of Deferred@19284714 after retry RPC after error@1857876693) -> (continuation of Deferred@2061251002 after retry RPC after error@324586504) -> (continuation of Deferred@542834075 after retry RPC after error@695834911) -> (continuation of Deferred@850607211 after retry RPC after error@1742638777) -> (continuation of Deferred@270471676 after retry RPC after error@1976288129) -> (continuation of Deferred@1580267831 after retry RPC after error@1292373774)))
21:05:04.704 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:05:04.705 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:05:04.705 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:04.705 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:04.706 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:04.706 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:04.706 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:04.706 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:04.706 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:05:04.724 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:05:04.725 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:05:04.725 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:04.725 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:04.725 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:04.725 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:04.725 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:04.726 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:04.727 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:05:04.744 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:05:04.745 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:05:04.745 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:04.745 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:04.745 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:04.745 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:04.745 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:04.745 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:04.746 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:05:04.764 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:05:04.764 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:05:04.765 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:04.765 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:04.765 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:04.765 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:04.765 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:04.765 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:04.766 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:05:04.784 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:05:04.784 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:05:04.784 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:04.784 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:04.784 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:04.784 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:04.785 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:04.785 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:04.785 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:05:04.804 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:05:04.805 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:05:04.805 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:04.805 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:04.805 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:04.805 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:04.805 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:04.805 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:04.806 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:05:04.884 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:05:04.885 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:05:04.885 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:04.885 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:04.885 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:04.885 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:04.886 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:04.886 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:04.886 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:05:04.887 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=182), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [23ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [79ms] refreshing cache from master, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [81ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [181ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [181ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [182ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, deferred=Deferred@1008358126(state=PENDING, result=null, callback=(continuation of Deferred@813383404 after retry RPC after error@1012508073) -> (continuation of Deferred@1413534326 after retry RPC after error@983787314) -> (continuation of Deferred@1405526740 after retry RPC after error@583217640) -> (continuation of Deferred@1891796132 after retry RPC after error@1375336313) -> (continuation of Deferred@927936400 after retry RPC after error@205001952) -> (continuation of Deferred@1748719207 after retry RPC after error@1994864166), errback=(continuation of Deferred@813383404 after retry RPC after error@1012508073) -> (continuation of Deferred@1413534326 after retry RPC after error@983787314) -> (continuation of Deferred@1405526740 after retry RPC after error@583217640) -> (continuation of Deferred@1891796132 after retry RPC after error@1375336313) -> (continuation of Deferred@927936400 after retry RPC after error@205001952) -> (continuation of Deferred@1748719207 after retry RPC after error@1994864166)))
21:05:04.887 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=182), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [23ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [79ms] refreshing cache from master, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [81ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [181ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [181ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [182ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, deferred=Deferred@1008358126(state=PENDING, result=null, callback=(continuation of Deferred@813383404 after retry RPC after error@1012508073) -> (continuation of Deferred@1413534326 after retry RPC after error@983787314) -> (continuation of Deferred@1405526740 after retry RPC after error@583217640) -> (continuation of Deferred@1891796132 after retry RPC after error@1375336313) -> (continuation of Deferred@927936400 after retry RPC after error@205001952) -> (continuation of Deferred@1748719207 after retry RPC after error@1994864166), errback=(continuation of Deferred@813383404 after retry RPC after error@1012508073) -> (continuation of Deferred@1413534326 after retry RPC after error@983787314) -> (continuation of Deferred@1405526740 after retry RPC after error@583217640) -> (continuation of Deferred@1891796132 after retry RPC after error@1375336313) -> (continuation of Deferred@927936400 after retry RPC after error@205001952) -> (continuation of Deferred@1748719207 after retry RPC after error@1994864166)))
21:05:04.904 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:05:04.904 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:05:04.905 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:04.905 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:04.905 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:04.905 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:04.905 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:04.905 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:04.906 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:05:04.924 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:05:04.924 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:05:04.925 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:04.925 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:04.925 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:04.925 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:04.925 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:04.925 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:04.926 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:05:04.944 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:05:04.944 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:05:04.944 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:04.945 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:04.945 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:04.945 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:04.945 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:04.945 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:04.946 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:05:04.964 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:05:04.965 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:05:04.965 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:04.965 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:04.965 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:04.965 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:04.965 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:04.965 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:04.966 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:05:04.984 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:05:04.984 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:05:04.985 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:04.985 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:04.985 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:04.985 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:04.985 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:04.985 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:04.985 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:04.985 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:04.987 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:05:04.987 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:05:05.024 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:05:05.024 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:05:05.025 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:05.025 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:05.025 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:05.025 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:05.025 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:05.025 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:05.026 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:05:05.044 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:05:05.044 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:05:05.045 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:05.045 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:05.045 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:05.045 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:05.045 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:05.045 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:05.046 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:05:05.104 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:05:05.104 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:05:05.105 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:05.105 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:05.105 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:05.105 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:05.105 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:05.105 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:05.106 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:05:05.106 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=202), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [39ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [81ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [83ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [141ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [141ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [141ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [201ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [201ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [202ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, deferred=Deferred@1256678989(state=PENDING, result=null, callback=(continuation of Deferred@204895359 after retry RPC after error@424909163) -> (continuation of Deferred@712857241 after retry RPC after error@1532594325) -> (continuation of Deferred@1693682189 after retry RPC after error@1634651961) -> (continuation of Deferred@560432147 after retry RPC after error@913161610) -> (continuation of Deferred@1320614568 after retry RPC after error@1496769344) -> (continuation of Deferred@1147970409 after retry RPC after error@532076419) -> (continuation of Deferred@1714167143 after retry RPC after error@141483521), errback=(continuation of Deferred@204895359 after retry RPC after error@424909163) -> (continuation of Deferred@712857241 after retry RPC after error@1532594325) -> (continuation of Deferred@1693682189 after retry RPC after error@1634651961) -> (continuation of Deferred@560432147 after retry RPC after error@913161610) -> (continuation of Deferred@1320614568 after retry RPC after error@1496769344) -> (continuation of Deferred@1147970409 after retry RPC after error@532076419) -> (continuation of Deferred@1714167143 after retry RPC after error@141483521)))
21:05:05.107 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=202), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [39ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [81ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [83ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [141ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [141ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [141ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [201ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [201ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [202ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, deferred=Deferred@1256678989(state=PENDING, result=null, callback=(continuation of Deferred@204895359 after retry RPC after error@424909163) -> (continuation of Deferred@712857241 after retry RPC after error@1532594325) -> (continuation of Deferred@1693682189 after retry RPC after error@1634651961) -> (continuation of Deferred@560432147 after retry RPC after error@913161610) -> (continuation of Deferred@1320614568 after retry RPC after error@1496769344) -> (continuation of Deferred@1147970409 after retry RPC after error@532076419) -> (continuation of Deferred@1714167143 after retry RPC after error@141483521), errback=(continuation of Deferred@204895359 after retry RPC after error@424909163) -> (continuation of Deferred@712857241 after retry RPC after error@1532594325) -> (continuation of Deferred@1693682189 after retry RPC after error@1634651961) -> (continuation of Deferred@560432147 after retry RPC after error@913161610) -> (continuation of Deferred@1320614568 after retry RPC after error@1496769344) -> (continuation of Deferred@1147970409 after retry RPC after error@532076419) -> (continuation of Deferred@1714167143 after retry RPC after error@141483521)))
21:05:05.125 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:05:05.125 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:05:05.125 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:05.125 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:05.125 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:05.126 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:05.126 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:05.126 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:05.126 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:05:05.144 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:05:05.144 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:05:05.145 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:05.145 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:05.145 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:05.145 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:05.145 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:05.145 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:05.146 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:05:05.164 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:05:05.164 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:05:05.165 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:05.165 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:05.165 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:05.165 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:05.165 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:05.165 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:05.166 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:05:05.184 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:05:05.184 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:05:05.185 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:05.185 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:05.185 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:05.185 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:05.185 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:05.185 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:05.186 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:05:05.204 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:05:05.204 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:05:05.204 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:05.204 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:05.204 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:05.204 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:05.205 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:05.205 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:05.205 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:05:05.244 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:05:05.245 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:05:05.245 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:05.245 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:05.245 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:05.245 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:05.246 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:05.246 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:05.246 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:05:05.304 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:05:05.305 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:05:05.305 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:05.305 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:05.305 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:05.305 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:05.305 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:05.305 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:05.306 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:05:05.307 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=181), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [79ms] refreshing cache from master, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [80ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [81ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [181ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [181ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [182ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, deferred=Deferred@1215105966(state=PENDING, result=null, callback=(continuation of Deferred@1087716267 after retry RPC after error@749635454) -> (continuation of Deferred@1195096692 after retry RPC after error@1739323718) -> (continuation of Deferred@248155771 after retry RPC after error@1451720107) -> (continuation of Deferred@1291158805 after retry RPC after error@1949053956) -> (continuation of Deferred@776983980 after retry RPC after error@1128828907) -> (continuation of Deferred@603285434 after retry RPC after error@913107924), errback=(continuation of Deferred@1087716267 after retry RPC after error@749635454) -> (continuation of Deferred@1195096692 after retry RPC after error@1739323718) -> (continuation of Deferred@248155771 after retry RPC after error@1451720107) -> (continuation of Deferred@1291158805 after retry RPC after error@1949053956) -> (continuation of Deferred@776983980 after retry RPC after error@1128828907) -> (continuation of Deferred@603285434 after retry RPC after error@913107924)))
21:05:05.307 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=181), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [79ms] refreshing cache from master, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [80ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [81ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [181ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [181ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [182ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, deferred=Deferred@1215105966(state=PENDING, result=null, callback=(continuation of Deferred@1087716267 after retry RPC after error@749635454) -> (continuation of Deferred@1195096692 after retry RPC after error@1739323718) -> (continuation of Deferred@248155771 after retry RPC after error@1451720107) -> (continuation of Deferred@1291158805 after retry RPC after error@1949053956) -> (continuation of Deferred@776983980 after retry RPC after error@1128828907) -> (continuation of Deferred@603285434 after retry RPC after error@913107924), errback=(continuation of Deferred@1087716267 after retry RPC after error@749635454) -> (continuation of Deferred@1195096692 after retry RPC after error@1739323718) -> (continuation of Deferred@248155771 after retry RPC after error@1451720107) -> (continuation of Deferred@1291158805 after retry RPC after error@1949053956) -> (continuation of Deferred@776983980 after retry RPC after error@1128828907) -> (continuation of Deferred@603285434 after retry RPC after error@913107924)))
21:05:05.324 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:05:05.324 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:05:05.325 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:05.325 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:05.325 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:05.325 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:05.325 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:05.325 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:05.326 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:05:05.344 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:05:05.344 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:05:05.345 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:05.345 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:05.345 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:05.345 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:05.345 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:05.345 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:05.346 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:05:05.364 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:05:05.364 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:05:05.364 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:05.364 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:05.364 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:05.364 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:05.365 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:05.365 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:05.366 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:05:05.384 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:05:05.384 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:05:05.385 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:05.385 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:05.385 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:05.385 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:05.385 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:05.385 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:05.386 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:05:05.404 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:05:05.404 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:05:05.405 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:05.405 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:05.405 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:05.405 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:05.405 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:05.405 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:05.406 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:05:05.444 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:05:05.444 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:05:05.445 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:05.445 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:05.445 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:05.445 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:05.445 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:05.445 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:05.446 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:05:05.484 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:05:05.484 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:05:05.484 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:05.484 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:05.485 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:05.485 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:05.485 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:05.485 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:05.486 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:05:05.524 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:05:05.524 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:05:05.524 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:05.524 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:05.524 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:05.524 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:05.525 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:05.525 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:05.526 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:05:05.526 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=201), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [39ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [40ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [161ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [161ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [161ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [199ms] refreshing cache from master, [199ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [200ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [201ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [201ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, deferred=Deferred@805183330(state=PENDING, result=null, callback=(continuation of Deferred@952796435 after retry RPC after error@1048517250) -> (continuation of Deferred@1834376558 after retry RPC after error@839670441) -> (continuation of Deferred@1585619942 after retry RPC after error@418318782) -> (continuation of Deferred@1200458464 after retry RPC after error@386156970) -> (continuation of Deferred@1557284159 after retry RPC after error@1016075746) -> (continuation of Deferred@392253978 after retry RPC after error@572330053) -> (continuation of Deferred@1814577796 after retry RPC after error@1345025510), errback=(continuation of Deferred@952796435 after retry RPC after error@1048517250) -> (continuation of Deferred@1834376558 after retry RPC after error@839670441) -> (continuation of Deferred@1585619942 after retry RPC after error@418318782) -> (continuation of Deferred@1200458464 after retry RPC after error@386156970) -> (continuation of Deferred@1557284159 after retry RPC after error@1016075746) -> (continuation of Deferred@392253978 after retry RPC after error@572330053) -> (continuation of Deferred@1814577796 after retry RPC after error@1345025510)))
21:05:05.527 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=201), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [39ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [40ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [161ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [161ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [161ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [199ms] refreshing cache from master, [199ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [200ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [201ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [201ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, deferred=Deferred@805183330(state=PENDING, result=null, callback=(continuation of Deferred@952796435 after retry RPC after error@1048517250) -> (continuation of Deferred@1834376558 after retry RPC after error@839670441) -> (continuation of Deferred@1585619942 after retry RPC after error@418318782) -> (continuation of Deferred@1200458464 after retry RPC after error@386156970) -> (continuation of Deferred@1557284159 after retry RPC after error@1016075746) -> (continuation of Deferred@392253978 after retry RPC after error@572330053) -> (continuation of Deferred@1814577796 after retry RPC after error@1345025510), errback=(continuation of Deferred@952796435 after retry RPC after error@1048517250) -> (continuation of Deferred@1834376558 after retry RPC after error@839670441) -> (continuation of Deferred@1585619942 after retry RPC after error@418318782) -> (continuation of Deferred@1200458464 after retry RPC after error@386156970) -> (continuation of Deferred@1557284159 after retry RPC after error@1016075746) -> (continuation of Deferred@392253978 after retry RPC after error@572330053) -> (continuation of Deferred@1814577796 after retry RPC after error@1345025510)))
21:05:05.544 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:05:05.545 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:05:05.545 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:05.545 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:05.545 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:05.545 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:05.545 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:05.545 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:05.546 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:05:05.564 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:05:05.564 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:05:05.565 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:05.565 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:05.565 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:05.565 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:05.565 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:05.565 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:05.566 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:05:05.584 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:05:05.584 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:05:05.584 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:05.584 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:05.585 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:05.585 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:05.585 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:05.585 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:05.586 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:05:05.604 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:05:05.605 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:05:05.605 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:05.605 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:05.605 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:05.605 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:05.605 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:05.605 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:05.606 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:05:05.624 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:05:05.624 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:05:05.625 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:05.625 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:05.625 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:05.625 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:05.625 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:05.625 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:05.626 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:05:05.664 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:05:05.664 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:05:05.665 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:05.665 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:05.665 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:05.665 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:05.665 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:05.665 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:05.666 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:05:05.704 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:05:05.704 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:05:05.704 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:05.705 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:05.705 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:05.705 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:05.705 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:05.705 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:05.706 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:05:05.706 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=161), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [39ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [161ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [161ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [161ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, deferred=Deferred@1008879054(state=PENDING, result=null, callback=(continuation of Deferred@2091749123 after retry RPC after error@693165228) -> (continuation of Deferred@1913379505 after retry RPC after error@1546255627) -> (continuation of Deferred@1282188314 after retry RPC after error@1668719751) -> (continuation of Deferred@1325675530 after retry RPC after error@1549162820) -> (continuation of Deferred@1966308000 after retry RPC after error@590850589) -> (continuation of Deferred@1308458517 after retry RPC after error@588219031), errback=(continuation of Deferred@2091749123 after retry RPC after error@693165228) -> (continuation of Deferred@1913379505 after retry RPC after error@1546255627) -> (continuation of Deferred@1282188314 after retry RPC after error@1668719751) -> (continuation of Deferred@1325675530 after retry RPC after error@1549162820) -> (continuation of Deferred@1966308000 after retry RPC after error@590850589) -> (continuation of Deferred@1308458517 after retry RPC after error@588219031)))
21:05:05.707 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=161), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [39ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [161ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [161ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [161ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, deferred=Deferred@1008879054(state=PENDING, result=null, callback=(continuation of Deferred@2091749123 after retry RPC after error@693165228) -> (continuation of Deferred@1913379505 after retry RPC after error@1546255627) -> (continuation of Deferred@1282188314 after retry RPC after error@1668719751) -> (continuation of Deferred@1325675530 after retry RPC after error@1549162820) -> (continuation of Deferred@1966308000 after retry RPC after error@590850589) -> (continuation of Deferred@1308458517 after retry RPC after error@588219031), errback=(continuation of Deferred@2091749123 after retry RPC after error@693165228) -> (continuation of Deferred@1913379505 after retry RPC after error@1546255627) -> (continuation of Deferred@1282188314 after retry RPC after error@1668719751) -> (continuation of Deferred@1325675530 after retry RPC after error@1549162820) -> (continuation of Deferred@1966308000 after retry RPC after error@590850589) -> (continuation of Deferred@1308458517 after retry RPC after error@588219031)))
21:05:05.724 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:05:05.725 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:05:05.725 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:05.725 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:05.725 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:05.725 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:05.726 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:05.726 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:05.727 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:05:05.744 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:05:05.744 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:05:05.745 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:05.745 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:05.746 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:05.746 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:05.746 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:05.746 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:05.747 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:05:05.764 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:05:05.764 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:05:05.765 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:05.765 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:05.765 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:05.765 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:05.765 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:05.765 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:05.766 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:05:05.784 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:05:05.784 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:05:05.784 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:05.784 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:05.785 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:05.785 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:05.785 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:05.785 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:05.786 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:05:05.804 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:05:05.805 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:05:05.805 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:05.805 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:05.805 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:05.805 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:05.805 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:05.805 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:05.806 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:05:05.824 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:05:05.824 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:05:05.825 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:05.825 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:05.825 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:05.825 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:05.825 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:05.825 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:05.826 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:05:05.884 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:05:05.885 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:05:05.885 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:05.885 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:05.885 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:05.885 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:05.885 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:05.885 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:05.886 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:05:05.944 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:05:05.945 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:05:05.945 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:05.945 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:05.945 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:05.945 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:05.945 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:05.945 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:05.946 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:05:05.947 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=222), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [23ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [161ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [161ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [162ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [163ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [221ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [221ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [222ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [222ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, deferred=Deferred@642291717(state=PENDING, result=null, callback=(continuation of Deferred@1421163119 after retry RPC after error@1734225034) -> (continuation of Deferred@1916642267 after retry RPC after error@274175273) -> (continuation of Deferred@1833739022 after retry RPC after error@2078341201) -> (continuation of Deferred@1683795853 after retry RPC after error@1217302770) -> (continuation of Deferred@1079119422 after retry RPC after error@1007440917) -> (continuation of Deferred@1026405598 after retry RPC after error@26786441) -> (continuation of Deferred@1365997434 after retry RPC after error@261696958), errback=(continuation of Deferred@1421163119 after retry RPC after error@1734225034) -> (continuation of Deferred@1916642267 after retry RPC after error@274175273) -> (continuation of Deferred@1833739022 after retry RPC after error@2078341201) -> (continuation of Deferred@1683795853 after retry RPC after error@1217302770) -> (continuation of Deferred@1079119422 after retry RPC after error@1007440917) -> (continuation of Deferred@1026405598 after retry RPC after error@26786441) -> (continuation of Deferred@1365997434 after retry RPC after error@261696958)))
21:05:05.948 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=222), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [23ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [161ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [161ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [162ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [163ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [221ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [221ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [222ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [222ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, deferred=Deferred@642291717(state=PENDING, result=null, callback=(continuation of Deferred@1421163119 after retry RPC after error@1734225034) -> (continuation of Deferred@1916642267 after retry RPC after error@274175273) -> (continuation of Deferred@1833739022 after retry RPC after error@2078341201) -> (continuation of Deferred@1683795853 after retry RPC after error@1217302770) -> (continuation of Deferred@1079119422 after retry RPC after error@1007440917) -> (continuation of Deferred@1026405598 after retry RPC after error@26786441) -> (continuation of Deferred@1365997434 after retry RPC after error@261696958), errback=(continuation of Deferred@1421163119 after retry RPC after error@1734225034) -> (continuation of Deferred@1916642267 after retry RPC after error@274175273) -> (continuation of Deferred@1833739022 after retry RPC after error@2078341201) -> (continuation of Deferred@1683795853 after retry RPC after error@1217302770) -> (continuation of Deferred@1079119422 after retry RPC after error@1007440917) -> (continuation of Deferred@1026405598 after retry RPC after error@26786441) -> (continuation of Deferred@1365997434 after retry RPC after error@261696958)))
21:05:05.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:05:05.954692 24071 master_runner.cc:186] Error getting master registration for 127.22.168.62:37239: Network error: Client connection negotiation failed: client connection to 127.22.168.62:37239: connect: Connection refused (error 111),
21:05:05.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:05:05.954907 24071 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
21:05:05.964 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:05:05.964 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:05:05.965 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:05.965 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:05.965 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:05.965 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:05.965 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:05.965 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:05.966 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:05:05.984 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:05:05.984 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:05:05.985 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:05.985 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:05.985 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:05.985 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:05.985 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:05.985 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:05.986 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:05:06.004 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:05:06.004 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:05:06.004 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:06.004 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:06.005 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:06.005 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:06.005 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:06.005 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:06.006 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:05:06.024 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:05:06.025 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:05:06.025 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:06.025 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:06.025 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:06.025 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:06.025 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:06.026 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:06.026 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:05:06.044 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:05:06.044 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:05:06.045 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:06.045 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:06.045 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:06.045 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:06.045 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:06.045 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:06.046 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:05:06.084 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:05:06.084 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:05:06.085 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:06.085 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:06.085 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:06.085 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:06.085 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:06.085 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:06.086 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:05:06.124 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:05:06.124 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:05:06.125 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:06.125 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:06.125 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:06.125 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:06.125 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:06.125 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:06.126 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:05:06.126 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=162), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [39ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [161ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [161ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [162ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, deferred=Deferred@776175105(state=PENDING, result=null, callback=(continuation of Deferred@543186008 after retry RPC after error@550734868) -> (continuation of Deferred@819782879 after retry RPC after error@98187187) -> (continuation of Deferred@670706124 after retry RPC after error@460694027) -> (continuation of Deferred@1289246630 after retry RPC after error@409699873) -> (continuation of Deferred@1856600233 after retry RPC after error@1492839053) -> (continuation of Deferred@1452401491 after retry RPC after error@311377198), errback=(continuation of Deferred@543186008 after retry RPC after error@550734868) -> (continuation of Deferred@819782879 after retry RPC after error@98187187) -> (continuation of Deferred@670706124 after retry RPC after error@460694027) -> (continuation of Deferred@1289246630 after retry RPC after error@409699873) -> (continuation of Deferred@1856600233 after retry RPC after error@1492839053) -> (continuation of Deferred@1452401491 after retry RPC after error@311377198)))
21:05:06.127 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=162), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [39ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [161ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [161ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [162ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, deferred=Deferred@776175105(state=PENDING, result=null, callback=(continuation of Deferred@543186008 after retry RPC after error@550734868) -> (continuation of Deferred@819782879 after retry RPC after error@98187187) -> (continuation of Deferred@670706124 after retry RPC after error@460694027) -> (continuation of Deferred@1289246630 after retry RPC after error@409699873) -> (continuation of Deferred@1856600233 after retry RPC after error@1492839053) -> (continuation of Deferred@1452401491 after retry RPC after error@311377198), errback=(continuation of Deferred@543186008 after retry RPC after error@550734868) -> (continuation of Deferred@819782879 after retry RPC after error@98187187) -> (continuation of Deferred@670706124 after retry RPC after error@460694027) -> (continuation of Deferred@1289246630 after retry RPC after error@409699873) -> (continuation of Deferred@1856600233 after retry RPC after error@1492839053) -> (continuation of Deferred@1452401491 after retry RPC after error@311377198)))
21:05:06.144 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:05:06.144 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:05:06.145 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:06.145 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:06.145 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:06.145 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:06.145 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:06.145 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:06.146 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:05:06.165 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:05:06.165 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:05:06.165 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:06.165 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:06.165 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:06.165 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:06.165 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:06.165 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:06.166 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:05:06.184 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:05:06.184 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:05:06.185 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:06.185 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:06.185 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:06.185 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:06.185 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:06.185 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:06.186 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:05:06.204 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:05:06.204 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:05:06.204 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:06.205 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:06.205 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:06.205 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:06.205 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:06.205 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:06.206 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:05:06.224 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:05:06.225 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:05:06.225 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:06.225 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:06.225 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:06.225 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:06.225 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:06.225 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:06.226 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:05:06.264 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:05:06.265 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:05:06.265 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:06.265 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:06.266 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:06.265 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:06.266 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:06.266 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:06.267 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:05:06.325 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:05:06.325 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:05:06.325 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:06.325 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:06.326 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:06.326 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:06.326 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:06.326 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:06.326 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:05:06.344 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:05:06.345 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:05:06.345 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:06.345 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:06.345 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:06.345 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:06.345 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:06.345 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:06.346 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:05:06.364 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:05:06.364 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:05:06.365 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:06.365 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:06.365 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:06.365 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:06.365 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:06.366 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:06.366 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:05:06.366 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=9, TimeoutTracker(timeout=250, elapsed=222), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [182ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [182ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [182ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [183ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [201ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [201ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [202ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [221ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [221ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [222ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [222ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, deferred=Deferred@844949494(state=PENDING, result=null, callback=(continuation of Deferred@1016891204 after retry RPC after error@735912143) -> (continuation of Deferred@1002629643 after retry RPC after error@1861536210) -> (continuation of Deferred@906679791 after retry RPC after error@1947418809) -> (continuation of Deferred@1479372418 after retry RPC after error@283088396) -> (continuation of Deferred@1889176661 after retry RPC after error@509746295) -> (continuation of Deferred@763897927 after retry RPC after error@1099674636) -> (continuation of Deferred@1708613708 after retry RPC after error@1809340709) -> (continuation of Deferred@125734409 after retry RPC after error@457797950), errback=(continuation of Deferred@1016891204 after retry RPC after error@735912143) -> (continuation of Deferred@1002629643 after retry RPC after error@1861536210) -> (continuation of Deferred@906679791 after retry RPC after error@1947418809) -> (continuation of Deferred@1479372418 after retry RPC after error@283088396) -> (continuation of Deferred@1889176661 after retry RPC after error@509746295) -> (continuation of Deferred@763897927 after retry RPC after error@1099674636) -> (continuation of Deferred@1708613708 after retry RPC after error@1809340709) -> (continuation of Deferred@125734409 after retry RPC after error@457797950)))
21:05:06.367 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=9, TimeoutTracker(timeout=250, elapsed=222), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [182ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [182ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [182ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [183ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [201ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [201ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [202ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [221ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [221ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [222ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [222ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, deferred=Deferred@844949494(state=PENDING, result=null, callback=(continuation of Deferred@1016891204 after retry RPC after error@735912143) -> (continuation of Deferred@1002629643 after retry RPC after error@1861536210) -> (continuation of Deferred@906679791 after retry RPC after error@1947418809) -> (continuation of Deferred@1479372418 after retry RPC after error@283088396) -> (continuation of Deferred@1889176661 after retry RPC after error@509746295) -> (continuation of Deferred@763897927 after retry RPC after error@1099674636) -> (continuation of Deferred@1708613708 after retry RPC after error@1809340709) -> (continuation of Deferred@125734409 after retry RPC after error@457797950), errback=(continuation of Deferred@1016891204 after retry RPC after error@735912143) -> (continuation of Deferred@1002629643 after retry RPC after error@1861536210) -> (continuation of Deferred@906679791 after retry RPC after error@1947418809) -> (continuation of Deferred@1479372418 after retry RPC after error@283088396) -> (continuation of Deferred@1889176661 after retry RPC after error@509746295) -> (continuation of Deferred@763897927 after retry RPC after error@1099674636) -> (continuation of Deferred@1708613708 after retry RPC after error@1809340709) -> (continuation of Deferred@125734409 after retry RPC after error@457797950)))
21:05:06.384 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:05:06.384 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:05:06.385 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:06.385 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:06.385 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:06.385 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:06.385 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:06.385 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:06.386 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:05:06.404 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:05:06.404 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:05:06.404 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:06.405 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:06.405 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:06.405 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:06.405 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:06.405 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:06.406 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:05:06.425 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:05:06.426 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:06.426 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:05:06.426 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:06.426 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:06.426 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:06.427 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:06.427 [INFO - pool-40-thread-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:06.427 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:05:06.444 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:05:06.445 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:05:06.445 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:06.445 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:06.445 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:06.445 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:06.445 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:06.446 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:06.446 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:05:06.464 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:05:06.465 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:05:06.465 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:06.465 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:06.465 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:06.466 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:06.466 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:06.466 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:06.467 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:05:06.504 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:05:06.505 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:05:06.505 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:06.505 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:06.505 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:06.505 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:06.506 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:06.506 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:06.506 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:05:06.524 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:05:06.525 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:05:06.525 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:06.525 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:06.525 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:06.525 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:06.525 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:06.525 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:06.526 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:05:06.624 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:05:06.624 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:05:06.625 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:06.625 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:06.625 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:06.625 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:06.625 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:06.625 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:06.627 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:05:06.627 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=242), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [43ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [43ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [141ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [141ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [142ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [241ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [241ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [242ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [243ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, deferred=Deferred@2033441950(state=PENDING, result=null, callback=(continuation of Deferred@2120204689 after retry RPC after error@388459245) -> (continuation of Deferred@1893490248 after retry RPC after error@401386465) -> (continuation of Deferred@398435268 after retry RPC after error@996393948) -> (continuation of Deferred@702543078 after retry RPC after error@1537143199) -> (continuation of Deferred@827709611 after retry RPC after error@336429182) -> (continuation of Deferred@1109606109 after retry RPC after error@1855794447) -> (continuation of Deferred@1905402103 after retry RPC after error@165957835), errback=(continuation of Deferred@2120204689 after retry RPC after error@388459245) -> (continuation of Deferred@1893490248 after retry RPC after error@401386465) -> (continuation of Deferred@398435268 after retry RPC after error@996393948) -> (continuation of Deferred@702543078 after retry RPC after error@1537143199) -> (continuation of Deferred@827709611 after retry RPC after error@336429182) -> (continuation of Deferred@1109606109 after retry RPC after error@1855794447) -> (continuation of Deferred@1905402103 after retry RPC after error@165957835)))
21:05:06.627 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=242), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [22ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [43ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [43ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [141ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [141ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [142ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [241ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [241ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [242ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [243ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, deferred=Deferred@2033441950(state=PENDING, result=null, callback=(continuation of Deferred@2120204689 after retry RPC after error@388459245) -> (continuation of Deferred@1893490248 after retry RPC after error@401386465) -> (continuation of Deferred@398435268 after retry RPC after error@996393948) -> (continuation of Deferred@702543078 after retry RPC after error@1537143199) -> (continuation of Deferred@827709611 after retry RPC after error@336429182) -> (continuation of Deferred@1109606109 after retry RPC after error@1855794447) -> (continuation of Deferred@1905402103 after retry RPC after error@165957835), errback=(continuation of Deferred@2120204689 after retry RPC after error@388459245) -> (continuation of Deferred@1893490248 after retry RPC after error@401386465) -> (continuation of Deferred@398435268 after retry RPC after error@996393948) -> (continuation of Deferred@702543078 after retry RPC after error@1537143199) -> (continuation of Deferred@827709611 after retry RPC after error@336429182) -> (continuation of Deferred@1109606109 after retry RPC after error@1855794447) -> (continuation of Deferred@1905402103 after retry RPC after error@165957835)))
21:05:06.644 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:05:06.644 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:05:06.645 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:06.645 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:06.645 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:06.645 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:06.645 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:06.645 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:06.646 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:05:06.664 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:05:06.664 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:05:06.664 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:06.664 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:06.664 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:06.670 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:06.670 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:06.670 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:06.670 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:05:06.684 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:05:06.685 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:05:06.685 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:06.685 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:06.685 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:06.685 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:06.685 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:06.685 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:06.686 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:05:06.704 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:05:06.704 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:05:06.705 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:06.705 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:06.705 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:06.705 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:06.705 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:06.705 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:06.706 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:05:06.724 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:05:06.724 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:05:06.724 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:06.725 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:06.725 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:06.725 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:06.725 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:06.725 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:06.726 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:05:06.764 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:05:06.764 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:05:06.764 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:06.764 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:06.765 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:06.765 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:06.765 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:06.765 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:06.766 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:05:06.824 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:05:06.825 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:05:06.825 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:06.825 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:06.825 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:06.825 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:06.826 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:06.826 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:06.826 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:05:06.885 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:05:06.885 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:05:06.885 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:06.885 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:06.885 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:06.885 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:06.886 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:06.886 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:06.887 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:05:06.887 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=243), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [19ms] refreshing cache from master, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [20ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [26ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [26ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [26ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [119ms] refreshing cache from master, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [181ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [181ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [182ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [241ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [242ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [242ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [243ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, deferred=Deferred@863368518(state=PENDING, result=null, callback=(continuation of Deferred@1613729373 after retry RPC after error@1120709179) -> (continuation of Deferred@2057889204 after retry RPC after error@1718349922) -> (continuation of Deferred@2049623956 after retry RPC after error@1561339800) -> (continuation of Deferred@1621687499 after retry RPC after error@1501161564) -> (continuation of Deferred@1967358592 after retry RPC after error@691230551) -> (continuation of Deferred@249210910 after retry RPC after error@418880145) -> (continuation of Deferred@1900629133 after retry RPC after error@1123234844), errback=(continuation of Deferred@1613729373 after retry RPC after error@1120709179) -> (continuation of Deferred@2057889204 after retry RPC after error@1718349922) -> (continuation of Deferred@2049623956 after retry RPC after error@1561339800) -> (continuation of Deferred@1621687499 after retry RPC after error@1501161564) -> (continuation of Deferred@1967358592 after retry RPC after error@691230551) -> (continuation of Deferred@249210910 after retry RPC after error@418880145) -> (continuation of Deferred@1900629133 after retry RPC after error@1123234844)))
21:05:06.888 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=243), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [1ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [19ms] refreshing cache from master, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [20ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [26ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [26ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [26ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [119ms] refreshing cache from master, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [181ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [181ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [182ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [241ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [242ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [242ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [243ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, deferred=Deferred@863368518(state=PENDING, result=null, callback=(continuation of Deferred@1613729373 after retry RPC after error@1120709179) -> (continuation of Deferred@2057889204 after retry RPC after error@1718349922) -> (continuation of Deferred@2049623956 after retry RPC after error@1561339800) -> (continuation of Deferred@1621687499 after retry RPC after error@1501161564) -> (continuation of Deferred@1967358592 after retry RPC after error@691230551) -> (continuation of Deferred@249210910 after retry RPC after error@418880145) -> (continuation of Deferred@1900629133 after retry RPC after error@1123234844), errback=(continuation of Deferred@1613729373 after retry RPC after error@1120709179) -> (continuation of Deferred@2057889204 after retry RPC after error@1718349922) -> (continuation of Deferred@2049623956 after retry RPC after error@1561339800) -> (continuation of Deferred@1621687499 after retry RPC after error@1501161564) -> (continuation of Deferred@1967358592 after retry RPC after error@691230551) -> (continuation of Deferred@249210910 after retry RPC after error@418880145) -> (continuation of Deferred@1900629133 after retry RPC after error@1123234844)))
21:05:06.904 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.62:37239(127.22.168.62:37239)] connecting to peer
21:05:06.905 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.168.61:32947(127.22.168.61:32947)] connecting to peer
21:05:06.905 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:06.905 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.168.61:32947(127.22.168.61:32947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:06.905 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:06.905 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:06.905 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.62:37239(127.22.168.62:37239)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:06.905 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.61:32947: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947
21:05:06.905 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:06.906 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239
21:05:06.906 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:05:06.907 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:05:06.907 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=ConnectToMaster, tablet=null, attempt=36, TimeoutTracker(timeout=49980, elapsed=49980), Traces: [0ms] sending RPC to server master-127.22.168.62:37239, [0ms] delaying RPC due to: Illegal state: connection is terminated, [16ms] refreshing cache from master, [17ms] sending RPC to server master-127.22.168.62:37239, [17ms] sending RPC to server master-127.22.168.60:33247, [17ms] sending RPC to server master-127.22.168.61:32947, [18ms] received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [19ms] received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [20ms] received response from server master-127.22.168.60:33247: OK, [20ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [38ms] refreshing cache from master, [38ms] sending RPC to server master-127.22.168.62:37239, [38ms] sending RPC to server master-127.22.168.60:33247, [38ms] sending RPC to server master-127.22.168.61:32947, [39ms] received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [40ms] received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [40ms] received response from server master-127.22.168.60:33247: OK, [41ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [58ms] refreshing cache from master, [58ms] sending RPC to server master-127.22.168.62:37239, [59ms] sending RPC to server master-127.22.168.60:33247, [59ms] sending RPC to server master-127.22.168.61:32947, [59ms] received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [60ms] received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [61ms] received response from server master-127.22.168.60:33247: OK, [61ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [78ms] refreshing cache from master, [78ms] sending RPC to server master-127.22.168.62:37239, [78ms] sending RPC to server master-127.22.168.60:33247, [78ms] sending RPC to server master-127.22.168.61:32947, [79ms] received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [79ms] received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [80ms] received response from server master-127.22.168.60:33247: OK, [80ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [99ms] refreshing cache from master, [99ms] sending RPC to server master-127.22.168.62:37239, [99ms] sending RPC to server master-127.22.168.60:33247, [99ms] sending RPC to server master-127.22.168.61:32947, [99ms] received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [100ms] received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [101ms] received response from server master-127.22.168.60:33247: OK, [101ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [157ms] refreshing cache from master, [157ms] sending RPC to server master-127.22.168.62:37239, [158ms] sending RPC to server master-127.22.168.60:33247, [158ms] sending RPC to server master-127.22.168.61:32947, [159ms] received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [159ms] received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [160ms] received response from server master-127.22.168.60:33247: OK, [160ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [217ms] refreshing cache from master, [217ms] sending RPC to server master-127.22.168.62:37239, [218ms] sending RPC to server master-127.22.168.60:33247, [218ms] sending RPC to server master-127.22.168.61:32947, [219ms] received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [219ms] received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [220ms] received response from server master-127.22.168.60:33247: OK, [220ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [257ms] refreshing cache from master, [257ms] sending RPC to server master-127.22.168.62:37239, [258ms] sending RPC to server master-127.22.168.60:33247, [258ms] sending RPC to server master-127.22.168.61:32947, [259ms] received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [259ms] received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [260ms] received response from server master-127.22.168.60:33247: OK, [260ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [737ms] refreshing cache from master, [737ms] sending RPC to server master-127.22.168.62:37239, [737ms] sending RPC to server master-127.22.168.60:33247, [737ms] sending RPC to server master-127.22.168.61:32947, [738ms] received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [738ms] received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [740ms] received response from server master-127.22.168.60:33247: OK, [741ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [877ms] refreshing cache from master, [877ms] sending RPC to server master-127.22.168.62:37239, [878ms] sending RPC to server master-127.22.168.60:33247, [878ms] sending RPC to server master-127.22.168.61:32947, [879ms] received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [879ms] received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [880ms] received response from server master-127.22.168.60:33247: OK, [880ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [2637ms] refreshing cache from master, [2637ms] sending RPC to server master-127.22.168.62:37239, [2637ms] sending RPC to server master-127.22.168.60:33247, [2638ms] sending RPC to server master-127.22.168.61:32947, [2638ms] received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [2639ms] received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [2639ms] received response from server master-127.22.168.60:33247: OK, [2640ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [6517ms] refreshing cache from master, [6517ms] sending RPC to server master-127.22.168.62:37239, [6518ms] sending RPC to server master-127.22.168.60:33247, [6518ms] sending RPC to server master-127.22.168.61:32947, [6519ms] received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [6519ms] received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [6520ms] received response from server master-127.22.168.60:33247: OK, [6520ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [6917ms] refreshing cache from master, [6917ms] sending RPC to server master-127.22.168.62:37239, [6917ms] too many traces: truncated at 100 traces, deferred=Deferred@111395518(state=PENDING, result=null, callback=passthrough -> ConnectToMasterCB for 127.22.168.62:37239 -> (continuation of Deferred@1571505570 after retry RPC after error@1407664861) -> (continuation of Deferred@1076157508 after retry RPC after error@1485455564) -> (continuation of Deferred@1594591274 after retry RPC after error@907266236) -> (continuation of Deferred@1345582278 after retry RPC after error@1513439788) -> (continuation of Deferred@1509743512 after retry RPC after error@1896005171) -> (continuation of Deferred@1054400785 after retry RPC after error@1309466820) -> (continuation of Deferred@1897120390 after retry RPC after error@360974769) -> (continuation of Deferred@1823318649 after retry RPC after error@1158571496) -> (continuation of Deferred@711362621 after retry RPC after error@521852225) -> (continuation of Deferred@514130609 after retry RPC after error@674366888) -> (continuation of Deferred@915101683 after retry RPC after error@2146581694) -> (continuation of Deferred@1232345467 after retry RPC after error@1901288120) -> (continuation of Deferred@487309230 after retry RPC after error@1936881533) -> (continuation of Deferred@446961577 after retry RPC after error@1900615903) -> (continuation of Deferred@830422935 after retry RPC after error@1373007706) -> (continuation of Deferred@1802838934 after retry RPC after error@1380143559) -> (continuation of Deferred@436756383 after retry RPC after error@2017877819) -> (continuation of Deferred@980265526 after retry RPC after error@1927809493) -> (continuation of Deferred@717509808 after retry RPC after error@1613855060) -> (continuation of Deferred@928662398 after retry RPC after error@97407713) -> (continuation of Deferred@220723004 after retry RPC after error@1723323413) -> (continuation of Deferred@59976206 after retry RPC after error@1950165447) -> (continuation of Deferred@1183956587 after retry RPC after error@509529552) -> (continuation of Deferred@67589967 after retry RPC after error@426321177) -> (continuation of Deferred@1379971769 after retry RPC after error@983094636) -> (continuation of Deferred@77283423 after retry RPC after error@1410006072) -> (continuation of Deferred@50878842 after retry RPC after error@2143801282) -> (continuation of Deferred@553705949 after retry RPC after error@585642745) -> (continuation of Deferred@1676311639 after retry RPC after error@571606583) -> (continuation of Deferred@70915489 after retry RPC after error@1730576421) -> (continuation of Deferred@1469993042 after retry RPC after error@1703921448) -> (continuation of Deferred@827080997 after retry RPC after error@1656277121) -> (continuation of Deferred@1191249988 after retry RPC after error@1554788619) -> (continuation of Deferred@263853778 after retry RPC after error@727853924), errback=org.apache.kudu.client.ConnectToCluster$1@646e4564 -> ConnectToMasterErrCB for 127.22.168.62:37239 -> (continuation of Deferred@1571505570 after retry RPC after error@1407664861) -> (continuation of Deferred@1076157508 after retry RPC after error@1485455564) -> (continuation of Deferred@1594591274 after retry RPC after error@907266236) -> (continuation of Deferred@1345582278 after retry RPC after error@1513439788) -> (continuation of Deferred@1509743512 after retry RPC after error@1896005171) -> (continuation of Deferred@1054400785 after retry RPC after error@1309466820) -> (continuation of Deferred@1897120390 after retry RPC after error@360974769) -> (continuation of Deferred@1823318649 after retry RPC after error@1158571496) -> (continuation of Deferred@711362621 after retry RPC after error@521852225) -> (continuation of Deferred@514130609 after retry RPC after error@674366888) -> (continuation of Deferred@915101683 after retry RPC after error@2146581694) -> (continuation of Deferred@1232345467 after retry RPC after error@1901288120) -> (continuation of Deferred@487309230 after retry RPC after error@1936881533) -> (continuation of Deferred@446961577 after retry RPC after error@1900615903) -> (continuation of Deferred@830422935 after retry RPC after error@1373007706) -> (continuation of Deferred@1802838934 after retry RPC after error@1380143559) -> (continuation of Deferred@436756383 after retry RPC after error@2017877819) -> (continuation of Deferred@980265526 after retry RPC after error@1927809493) -> (continuation of Deferred@717509808 after retry RPC after error@1613855060) -> (continuation of Deferred@928662398 after retry RPC after error@97407713) -> (continuation of Deferred@220723004 after retry RPC after error@1723323413) -> (continuation of Deferred@59976206 after retry RPC after error@1950165447) -> (continuation of Deferred@1183956587 after retry RPC after error@509529552) -> (continuation of Deferred@67589967 after retry RPC after error@426321177) -> (continuation of Deferred@1379971769 after retry RPC after error@983094636) -> (continuation of Deferred@77283423 after retry RPC after error@1410006072) -> (continuation of Deferred@50878842 after retry RPC after error@2143801282) -> (continuation of Deferred@553705949 after retry RPC after error@585642745) -> (continuation of Deferred@1676311639 after retry RPC after error@571606583) -> (continuation of Deferred@70915489 after retry RPC after error@1730576421) -> (continuation of Deferred@1469993042 after retry RPC after error@1703921448) -> (continuation of Deferred@827080997 after retry RPC after error@1656277121) -> (continuation of Deferred@1191249988 after retry RPC after error@1554788619) -> (continuation of Deferred@263853778 after retry RPC after error@727853924)))
21:05:06.908 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.168.62:37239: cannot complete before timeout: KuduRpc(method=ConnectToMaster, tablet=null, attempt=36, TimeoutTracker(timeout=49980, elapsed=49980), Traces: [0ms] sending RPC to server master-127.22.168.62:37239, [0ms] delaying RPC due to: Illegal state: connection is terminated, [16ms] refreshing cache from master, [17ms] sending RPC to server master-127.22.168.62:37239, [17ms] sending RPC to server master-127.22.168.60:33247, [17ms] sending RPC to server master-127.22.168.61:32947, [18ms] received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [19ms] received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [20ms] received response from server master-127.22.168.60:33247: OK, [20ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [38ms] refreshing cache from master, [38ms] sending RPC to server master-127.22.168.62:37239, [38ms] sending RPC to server master-127.22.168.60:33247, [38ms] sending RPC to server master-127.22.168.61:32947, [39ms] received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [40ms] received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [40ms] received response from server master-127.22.168.60:33247: OK, [41ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [58ms] refreshing cache from master, [58ms] sending RPC to server master-127.22.168.62:37239, [59ms] sending RPC to server master-127.22.168.60:33247, [59ms] sending RPC to server master-127.22.168.61:32947, [59ms] received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [60ms] received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [61ms] received response from server master-127.22.168.60:33247: OK, [61ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [78ms] refreshing cache from master, [78ms] sending RPC to server master-127.22.168.62:37239, [78ms] sending RPC to server master-127.22.168.60:33247, [78ms] sending RPC to server master-127.22.168.61:32947, [79ms] received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [79ms] received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [80ms] received response from server master-127.22.168.60:33247: OK, [80ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [99ms] refreshing cache from master, [99ms] sending RPC to server master-127.22.168.62:37239, [99ms] sending RPC to server master-127.22.168.60:33247, [99ms] sending RPC to server master-127.22.168.61:32947, [99ms] received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [100ms] received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [101ms] received response from server master-127.22.168.60:33247: OK, [101ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [157ms] refreshing cache from master, [157ms] sending RPC to server master-127.22.168.62:37239, [158ms] sending RPC to server master-127.22.168.60:33247, [158ms] sending RPC to server master-127.22.168.61:32947, [159ms] received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [159ms] received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [160ms] received response from server master-127.22.168.60:33247: OK, [160ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [217ms] refreshing cache from master, [217ms] sending RPC to server master-127.22.168.62:37239, [218ms] sending RPC to server master-127.22.168.60:33247, [218ms] sending RPC to server master-127.22.168.61:32947, [219ms] received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [219ms] received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [220ms] received response from server master-127.22.168.60:33247: OK, [220ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [257ms] refreshing cache from master, [257ms] sending RPC to server master-127.22.168.62:37239, [258ms] sending RPC to server master-127.22.168.60:33247, [258ms] sending RPC to server master-127.22.168.61:32947, [259ms] received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [259ms] received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [260ms] received response from server master-127.22.168.60:33247: OK, [260ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [737ms] refreshing cache from master, [737ms] sending RPC to server master-127.22.168.62:37239, [737ms] sending RPC to server master-127.22.168.60:33247, [737ms] sending RPC to server master-127.22.168.61:32947, [738ms] received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [738ms] received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [740ms] received response from server master-127.22.168.60:33247: OK, [741ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [877ms] refreshing cache from master, [877ms] sending RPC to server master-127.22.168.62:37239, [878ms] sending RPC to server master-127.22.168.60:33247, [878ms] sending RPC to server master-127.22.168.61:32947, [879ms] received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [879ms] received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [880ms] received response from server master-127.22.168.60:33247: OK, [880ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [2637ms] refreshing cache from master, [2637ms] sending RPC to server master-127.22.168.62:37239, [2637ms] sending RPC to server master-127.22.168.60:33247, [2638ms] sending RPC to server master-127.22.168.61:32947, [2638ms] received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [2639ms] received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [2639ms] received response from server master-127.22.168.60:33247: OK, [2640ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [6517ms] refreshing cache from master, [6517ms] sending RPC to server master-127.22.168.62:37239, [6518ms] sending RPC to server master-127.22.168.60:33247, [6518ms] sending RPC to server master-127.22.168.61:32947, [6519ms] received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [6519ms] received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [6520ms] received response from server master-127.22.168.60:33247: OK, [6520ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [6917ms] refreshing cache from master, [6917ms] sending RPC to server master-127.22.168.62:37239, [6917ms] too many traces: truncated at 100 traces, deferred=Deferred@111395518(state=PENDING, result=null, callback=passthrough -> ConnectToMasterCB for 127.22.168.62:37239 -> (continuation of Deferred@1571505570 after retry RPC after error@1407664861) -> (continuation of Deferred@1076157508 after retry RPC after error@1485455564) -> (continuation of Deferred@1594591274 after retry RPC after error@907266236) -> (continuation of Deferred@1345582278 after retry RPC after error@1513439788) -> (continuation of Deferred@1509743512 after retry RPC after error@1896005171) -> (continuation of Deferred@1054400785 after retry RPC after error@1309466820) -> (continuation of Deferred@1897120390 after retry RPC after error@360974769) -> (continuation of Deferred@1823318649 after retry RPC after error@1158571496) -> (continuation of Deferred@711362621 after retry RPC after error@521852225) -> (continuation of Deferred@514130609 after retry RPC after error@674366888) -> (continuation of Deferred@915101683 after retry RPC after error@2146581694) -> (continuation of Deferred@1232345467 after retry RPC after error@1901288120) -> (continuation of Deferred@487309230 after retry RPC after error@1936881533) -> (continuation of Deferred@446961577 after retry RPC after error@1900615903) -> (continuation of Deferred@830422935 after retry RPC after error@1373007706) -> (continuation of Deferred@1802838934 after retry RPC after error@1380143559) -> (continuation of Deferred@436756383 after retry RPC after error@2017877819) -> (continuation of Deferred@980265526 after retry RPC after error@1927809493) -> (continuation of Deferred@717509808 after retry RPC after error@1613855060) -> (continuation of Deferred@928662398 after retry RPC after error@97407713) -> (continuation of Deferred@220723004 after retry RPC after error@1723323413) -> (continuation of Deferred@59976206 after retry RPC after error@1950165447) -> (continuation of Deferred@1183956587 after retry RPC after error@509529552) -> (continuation of Deferred@67589967 after retry RPC after error@426321177) -> (continuation of Deferred@1379971769 after retry RPC after error@983094636) -> (continuation of Deferred@77283423 after retry RPC after error@1410006072) -> (continuation of Deferred@50878842 after retry RPC after error@2143801282) -> (continuation of Deferred@553705949 after retry RPC after error@585642745) -> (continuation of Deferred@1676311639 after retry RPC after error@571606583) -> (continuation of Deferred@70915489 after retry RPC after error@1730576421) -> (continuation of Deferred@1469993042 after retry RPC after error@1703921448) -> (continuation of Deferred@827080997 after retry RPC after error@1656277121) -> (continuation of Deferred@1191249988 after retry RPC after error@1554788619) -> (continuation of Deferred@263853778 after retry RPC after error@727853924), errback=org.apache.kudu.client.ConnectToCluster$1@646e4564 -> ConnectToMasterErrCB for 127.22.168.62:37239 -> (continuation of Deferred@1571505570 after retry RPC after error@1407664861) -> (continuation of Deferred@1076157508 after retry RPC after error@1485455564) -> (continuation of Deferred@1594591274 after retry RPC after error@907266236) -> (continuation of Deferred@1345582278 after retry RPC after error@1513439788) -> (continuation of Deferred@1509743512 after retry RPC after error@1896005171) -> (continuation of Deferred@1054400785 after retry RPC after error@1309466820) -> (continuation of Deferred@1897120390 after retry RPC after error@360974769) -> (continuation of Deferred@1823318649 after retry RPC after error@1158571496) -> (continuation of Deferred@711362621 after retry RPC after error@521852225) -> (continuation of Deferred@514130609 after retry RPC after error@674366888) -> (continuation of Deferred@915101683 after retry RPC after error@2146581694) -> (continuation of Deferred@1232345467 after retry RPC after error@1901288120) -> (continuation of Deferred@487309230 after retry RPC after error@1936881533) -> (continuation of Deferred@446961577 after retry RPC after error@1900615903) -> (continuation of Deferred@830422935 after retry RPC after error@1373007706) -> (continuation of Deferred@1802838934 after retry RPC after error@1380143559) -> (continuation of Deferred@436756383 after retry RPC after error@2017877819) -> (continuation of Deferred@980265526 after retry RPC after error@1927809493) -> (continuation of Deferred@717509808 after retry RPC after error@1613855060) -> (continuation of Deferred@928662398 after retry RPC after error@97407713) -> (continuation of Deferred@220723004 after retry RPC after error@1723323413) -> (continuation of Deferred@59976206 after retry RPC after error@1950165447) -> (continuation of Deferred@1183956587 after retry RPC after error@509529552) -> (continuation of Deferred@67589967 after retry RPC after error@426321177) -> (continuation of Deferred@1379971769 after retry RPC after error@983094636) -> (continuation of Deferred@77283423 after retry RPC after error@1410006072) -> (continuation of Deferred@50878842 after retry RPC after error@2143801282) -> (continuation of Deferred@553705949 after retry RPC after error@585642745) -> (continuation of Deferred@1676311639 after retry RPC after error@571606583) -> (continuation of Deferred@70915489 after retry RPC after error@1730576421) -> (continuation of Deferred@1469993042 after retry RPC after error@1703921448) -> (continuation of Deferred@827080997 after retry RPC after error@1656277121) -> (continuation of Deferred@1191249988 after retry RPC after error@1554788619) -> (continuation of Deferred@263853778 after retry RPC after error@727853924)))
21:05:06.909 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947; will retry
21:05:06.909 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=GetTransactionState, tablet=null, attempt=2, TimeoutTracker(timeout=50000, elapsed=50003), Traces: [0ms] refreshing cache from master, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [2ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [2ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [4ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [4ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [21ms] refreshing cache from master, [22ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [22ms] Sub RPC ConnectToMaster: delaying RPC due to: Illegal state: connection is terminated, [22ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [22ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [23ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [23ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [38ms] Sub RPC ConnectToMaster: refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [40ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [42ms] Sub RPC ConnectToMaster: delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [60ms] Sub RPC ConnectToMaster: refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [63ms] Sub RPC ConnectToMaster: delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [80ms] Sub RPC ConnectToMaster: refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [83ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [83ms] Sub RPC ConnectToMaster: delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [100ms] Sub RPC ConnectToMaster: refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [102ms] Sub RPC ConnectToMaster: delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [121ms] Sub RPC ConnectToMaster: refreshing cache from master, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [123ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [123ms] Sub RPC ConnectToMaster: delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [179ms] Sub RPC ConnectToMaster: refreshing cache from master, [179ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [181ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [181ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [182ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [182ms] Sub RPC ConnectToMaster: delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [239ms] Sub RPC ConnectToMaster: refreshing cache from master, [239ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [241ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [241ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [242ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [242ms] Sub RPC ConnectToMaster: delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [279ms] Sub RPC ConnectToMaster: refreshing cache from master, [279ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [280ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [280ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [281ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [281ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [282ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [282ms] Sub RPC ConnectToMaster: delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [759ms] Sub RPC ConnectToMaster: refreshing cache from master, [759ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [759ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [759ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [760ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [760ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [762ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [763ms] Sub RPC ConnectToMaster: delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [899ms] Sub RPC ConnectToMaster: refreshing cache from master, [899ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [900ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [900ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [901ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [901ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [902ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [902ms] Sub RPC ConnectToMaster: delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [2659ms] Sub RPC ConnectToMaster: refreshing cache from master, [2659ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [2659ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [2660ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [2660ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [2661ms] too many traces: truncated at 100 traces, deferred=Deferred@1500972171(state=PENDING, result=null, callback=(continuation of Deferred@111445489 after retry RPC after error@1017620298), errback=(continuation of Deferred@111445489 after retry RPC after error@1017620298)))
21:05:06.911 [DEBUG - main] (Connection.java:575) [peer master-127.22.168.62:37239(127.22.168.62:37239)] disconnecting while in state TERMINATED
21:05:06.911 [DEBUG - main] (Connection.java:575) [peer master-127.22.168.61:32947(127.22.168.61:32947)] disconnecting while in state TERMINATED
21:05:06.911 [DEBUG - main] (Connection.java:575) [peer master-127.22.168.60:33247(127.22.168.60:33247)] disconnecting while in state READY
21:05:06.911 [DEBUG - main] (Connection.java:575) [peer e24858bd4ab14759b2252ddad7f0e5cd(127.22.168.3:35191)] disconnecting while in state READY
21:05:06.912 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.22.168.60:33247(127.22.168.60:33247)] handling channelInactive
21:05:06.912 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.168.60:33247(127.22.168.60:33247)] cleaning up while in state READY due to: connection closed
21:05:06.912 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2704) Releasing all remaining resources
21:05:06.913 [DEBUG - kudu-nio-1] (Connection.java:239) [peer e24858bd4ab14759b2252ddad7f0e5cd(127.22.168.3:35191)] handling channelInactive
21:05:06.913 [DEBUG - kudu-nio-1] (Connection.java:709) [peer e24858bd4ab14759b2252ddad7f0e5cd(127.22.168.3:35191)] cleaning up while in state READY due to: connection closed
21:05:06.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:05:06.913662 23200 external_mini_cluster.cc:1699] Attempting to check leaks for /tmp/dist-test-taskRxIrUU/build/asan/bin/kudu pid 23494
21:05:06.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:05:06.928208 23648 generic_service.cc:196] Checking for leaks (request via RPC)
21:05:07.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:05:07.086156 23200 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskRxIrUU/build/asan/bin/kudu with pid 23494
21:05:07.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:05:07.112294 23815 connection.cc:537] server connection from 127.22.168.1:59601 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
21:05:07.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:05:07.112581 23675 connection.cc:537] client connection to 127.22.168.1:42241 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
21:05:07.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:05:07.112735 23200 external_mini_cluster.cc:1699] Attempting to check leaks for /tmp/dist-test-taskRxIrUU/build/asan/bin/kudu pid 23658
21:05:07.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:05:07.112833 23675 proxy.cc:239] Call had error, refreshing address and retrying: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
21:05:07.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:05:07.114871 23675 consensus_peers.cc:597] T 260c684b258041bab84c3ff36578f96b P 1acec06b763242d082ddb75231fc4eca -> Peer 210cdae6f0e44d83acf5641ce563b738 (127.22.168.1:42241): Couldn't send request to peer 210cdae6f0e44d83acf5641ce563b738. Status: Network error: Client connection negotiation failed: client connection to 127.22.168.1:42241: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
21:05:07.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:05:07.120182 23789 generic_service.cc:196] Checking for leaks (request via RPC)
21:05:07.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:05:07.129609 23817 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.22.168.1:42241: connect: Connection refused (error 111)
21:05:07.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:05:07.131213 23817 consensus_peers.cc:597] T 01aa0e192c434b468ec7ed90d48563b3 P e24858bd4ab14759b2252ddad7f0e5cd -> Peer 210cdae6f0e44d83acf5641ce563b738 (127.22.168.1:42241): Couldn't send request to peer 210cdae6f0e44d83acf5641ce563b738. Status: Network error: Client connection negotiation failed: client connection to 127.22.168.1:42241: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
21:05:07.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:05:07.231103 23200 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskRxIrUU/build/asan/bin/kudu with pid 23658
21:05:07.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:05:07.252090 23200 external_mini_cluster.cc:1699] Attempting to check leaks for /tmp/dist-test-taskRxIrUU/build/asan/bin/kudu pid 23798
21:05:07.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:05:07.253957 23817 consensus_peers.cc:597] T 01aa0e192c434b468ec7ed90d48563b3 P e24858bd4ab14759b2252ddad7f0e5cd -> Peer 1acec06b763242d082ddb75231fc4eca (127.22.168.2:37409): Couldn't send request to peer 1acec06b763242d082ddb75231fc4eca. Status: Network error: Client connection negotiation failed: client connection to 127.22.168.2:37409: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
21:05:07.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:05:07.259370 23931 generic_service.cc:196] Checking for leaks (request via RPC)
21:05:07.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:05:07.370999 23200 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskRxIrUU/build/asan/bin/kudu with pid 23798
21:05:07.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:05:07.391575 23200 external_mini_cluster.cc:1699] Attempting to check leaks for /tmp/dist-test-taskRxIrUU/build/asan/bin/kudu pid 24071
21:05:07.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:05:07.398236 24143 generic_service.cc:196] Checking for leaks (request via RPC)
21:05:07.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:05:07.566488 23200 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskRxIrUU/build/asan/bin/kudu with pid 24071
21:05:07.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-10-28T21:05:07Z chronyd exiting
21:05:07.733 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestKuduTransaction.testTxnKeepaliveRollingSwitchToOtherTxnManager: failed attempt 1
java.lang.AssertionError: Timed out: cannot complete before timeout: KuduRpc(method=GetTransactionState, tablet=null, attempt=2, TimeoutTracker(timeout=50000, elapsed=50003), Traces: [0ms] refreshing cache from master, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [2ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [2ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [4ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [4ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [21ms] refreshing cache from master, [22ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [22ms] Sub RPC ConnectToMaster: delaying RPC due to: Illegal state: connection is terminated, [22ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [22ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [23ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [23ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [38ms] Sub RPC ConnectToMaster: refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [40ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [42ms] Sub RPC ConnectToMaster: delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [60ms] Sub RPC ConnectToMaster: refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [63ms] Sub RPC ConnectToMaster: delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [80ms] Sub RPC ConnectToMaster: refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [83ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [83ms] Sub RPC ConnectToMaster: delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [100ms] Sub RPC ConnectToMaster: refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [102ms] Sub RPC ConnectToMaster: delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [121ms] Sub RPC ConnectToMaster: refreshing cache from master, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [123ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [123ms] Sub RPC ConnectToMaster: delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [179ms] Sub RPC ConnectToMaster: refreshing cache from master, [179ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [181ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [181ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [182ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [182ms] Sub RPC ConnectToMaster: delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [239ms] Sub RPC ConnectToMaster: refreshing cache from master, [239ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [241ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [241ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [242ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [242ms] Sub RPC ConnectToMaster: delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [279ms] Sub RPC ConnectToMaster: refreshing cache from master, [279ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [280ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [280ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [281ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [281ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [282ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [282ms] Sub RPC ConnectToMaster: delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [759ms] Sub RPC ConnectToMaster: refreshing cache from master, [759ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [759ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [759ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [760ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [760ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [762ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [763ms] Sub RPC ConnectToMaster: delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [899ms] Sub RPC ConnectToMaster: refreshing cache from master, [899ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [900ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [900ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [901ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [901ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [902ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [902ms] Sub RPC ConnectToMaster: delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [2659ms] Sub RPC ConnectToMaster: refreshing cache from master, [2659ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [2659ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [2660ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [2660ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [2661ms] too many traces: truncated at 100 traces, deferred=Deferred@1500972171(state=PENDING, result=null, callback=(continuation of Deferred@111445489 after retry RPC after error@1017620298), errback=(continuation of Deferred@111445489 after retry RPC after error@1017620298)))
at org.junit.Assert.fail(Assert.java:89) ~[junit-4.13.2.jar:4.13.2]
at org.junit.Assert.assertTrue(Assert.java:42) ~[junit-4.13.2.jar:4.13.2]
at org.apache.kudu.client.TestKuduTransaction.testTxnKeepaliveRollingSwitchToOtherTxnManager(TestKuduTransaction.java:1214) ~[test/:?]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_275]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_275]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_275]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_275]
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2]
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2]
at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_275]
at java.lang.Thread.run(Thread.java:748) ~[?:1.8.0_275]
java.lang.AssertionError: Timed out: cannot complete before timeout: KuduRpc(method=GetTransactionState, tablet=null, attempt=2, TimeoutTracker(timeout=50000, elapsed=50003), Traces: [0ms] refreshing cache from master, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [2ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [2ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [2ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [4ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [4ms] delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [21ms] refreshing cache from master, [22ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [22ms] Sub RPC ConnectToMaster: delaying RPC due to: Illegal state: connection is terminated, [22ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [22ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [23ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [23ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [38ms] Sub RPC ConnectToMaster: refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [40ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [42ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [42ms] Sub RPC ConnectToMaster: delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [60ms] Sub RPC ConnectToMaster: refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [62ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [63ms] Sub RPC ConnectToMaster: delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [80ms] Sub RPC ConnectToMaster: refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [82ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [83ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [83ms] Sub RPC ConnectToMaster: delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [100ms] Sub RPC ConnectToMaster: refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [102ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [102ms] Sub RPC ConnectToMaster: delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [121ms] Sub RPC ConnectToMaster: refreshing cache from master, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [121ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [123ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [123ms] Sub RPC ConnectToMaster: delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [179ms] Sub RPC ConnectToMaster: refreshing cache from master, [179ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [181ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [181ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [182ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [182ms] Sub RPC ConnectToMaster: delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [239ms] Sub RPC ConnectToMaster: refreshing cache from master, [239ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [241ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [241ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [242ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [242ms] Sub RPC ConnectToMaster: delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [279ms] Sub RPC ConnectToMaster: refreshing cache from master, [279ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [280ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [280ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [281ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [281ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [282ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [282ms] Sub RPC ConnectToMaster: delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [759ms] Sub RPC ConnectToMaster: refreshing cache from master, [759ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [759ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [759ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [760ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [760ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [762ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [763ms] Sub RPC ConnectToMaster: delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [899ms] Sub RPC ConnectToMaster: refreshing cache from master, [899ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [900ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [900ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [901ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.61:32947: Network error: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947, [901ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [902ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.60:33247: OK, [902ms] Sub RPC ConnectToMaster: delaying RPC due to: Service unavailable: Master config (127.22.168.62:37239,127.22.168.60:33247,127.22.168.61:32947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.61:32947(127.22.168.61:32947): Connection refused: /127.22.168.61:32947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [2659ms] Sub RPC ConnectToMaster: refreshing cache from master, [2659ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.62:37239, [2659ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.60:33247, [2660ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.168.61:32947, [2660ms] Sub RPC ConnectToMaster: received response from server master-127.22.168.62:37239: Network error: Failed to connect to peer master-127.22.168.62:37239(127.22.168.62:37239): Connection refused: /127.22.168.62:37239, [2661ms] too many traces: truncated at 100 traces, deferred=Deferred@1500972171(state=PENDING, result=null, callback=(continuation of Deferred@111445489 after retry RPC after error@1017620298), errback=(continuation of Deferred@111445489 after retry RPC after error@1017620298)))
at org.junit.Assert.fail(Assert.java:89)
at org.junit.Assert.assertTrue(Assert.java:42)
at org.apache.kudu.client.TestKuduTransaction.testTxnKeepaliveRollingSwitchToOtherTxnManager(TestKuduTransaction.java:1214)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.lang.Thread.run(Thread.java:748)